Skip to content

Commit f3f60b9

Browse files
sjpbsjpb
authored
Support manila fileshares (cephfs) (#344)
* add support for manila to common environment * use manila share for /scratch in stackhpc env w/ tests * make home volume optional in skeleton TF * remove share creation from CI TF * add manila UI for caas * support manila- or nfs/volume- based home dirs in caas * remove manila config from UI * add optional platform-lifecycle manila share for homedirs for caas * add home and project manila config for caas * tweak home volume size UI description to account for shares * fix caas manila config typo * tidy PR diff * bump fatimage to include manila client * Revert commit "tweak home volume size UI description to account for shares" This reverts commit 3d9cfba. * add manila UI for caas * add defaults for new caas manila extravars, where possible * make cluster_home_manila_share_type optional for when default share type is defined * address review comments * bump manila requirement after role release * default usage of home manila share to match project share for caas * remove caas manila-specific ui-meta
1 parent b902115 commit f3f60b9

File tree

17 files changed

+122
-25
lines changed

17 files changed

+122
-25
lines changed

‎ansible/fatimage.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,16 @@
4141
tasks_from: client-install.yml
4242
when: "'freeipa_client' in group_names"
4343

44-
# - import_playbook: filesystems.yml
45-
- name: nfs
44+
# - import_playbook: filesystems.yml:
45+
- name: Install nfs packages
4646
dnf:
4747
name: nfs-utils
48+
when: "'nfs' in group_names"
49+
- name: Install Manila client packages
50+
include_role:
51+
name: stackhpc.os-manila-mount
52+
tasks_from: install.yml
53+
when: "'manila' in group_names"
4854

4955
- import_playbook: extras.yml
5056

‎ansible/filesystems.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,11 @@
1616
tasks:
1717
- include_role:
1818
name: stackhpc.nfs
19+
20+
- name: Setup Manila share mounts
21+
hosts: manila
22+
become: true
23+
tags: manila
24+
tasks:
25+
- include_role:
26+
name: stackhpc.os-manila-mount

‎ansible/roles/cluster_infra/templates/resources.tf.j2‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,22 @@ resource "openstack_blockstorage_volume_v3" "state" {
7979
size = "{{ state_volume_size }}"
8080
}
8181

82+
{% if cluster_home_manila_share | bool %}
83+
resource "openstack_sharedfilesystem_share_v2" "home" {
84+
name = "{{ cluster_name }}-home"
85+
description = "Home for cluster"
86+
share_proto = "CEPHFS"
87+
share_type = {{ '"' + cluster_home_manila_share_type + '"' | default('null') }}
88+
size = "{{ home_volume_size }}"
89+
}
90+
91+
resource "openstack_sharedfilesystem_share_access_v2" "home" {
92+
share_id = openstack_sharedfilesystem_share_v2.home.id
93+
access_type = "cephx"
94+
access_to = "cluster_{{ cluster_id }}"
95+
access_level = "rw"
96+
}
97+
{% else %}
8298
resource "openstack_blockstorage_volume_v3" "home" {
8399
name = "{{ cluster_name }}-home"
84100
description = "Home for control node"
@@ -89,6 +105,7 @@ resource "openstack_blockstorage_volume_v3" "home" {
89105
{% endif %}
90106
{% endif %}
91107
}
108+
{% endif %}
92109

93110
######
94111
###### Cluster network
@@ -334,13 +351,15 @@ resource "openstack_compute_instance_v2" "control" {
334351
uuid = openstack_blockstorage_volume_v3.state.id
335352
}
336353

354+
{% if not cluster_home_manila_share | bool %}
337355
# home volume:
338356
block_device {
339357
destination_type = "volume"
340358
source_type = "volume"
341359
boot_index = -1
342360
uuid = openstack_blockstorage_volume_v3.home.id
343361
}
362+
{% endif %}
344363

345364
# Use cloud-init to a) inject SSH keys b) configure volumes
346365
user_data = <<-EOF
@@ -359,12 +378,14 @@ resource "openstack_compute_instance_v2" "control" {
359378
- {{ ssh_key }}
360379
{%- endfor %}
361380
bootcmd:
362-
%{for volume in [openstack_blockstorage_volume_v3.state, openstack_blockstorage_volume_v3.home]}
381+
%{for volume in [openstack_blockstorage_volume_v3.state, {% if not cluster_home_manila_share | bool %} openstack_blockstorage_volume_v3.home {% endif %}]}
363382
- BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV || mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
364383
%{endfor}
365384
mounts:
366385
- [LABEL=state, {{ appliances_state_dir }}, auto]
386+
{% if not cluster_home_manila_share | bool %}
367387
- [LABEL=home, /exports/home, auto]
388+
{% endif %}
368389
EOF
369390
}
370391

‎environments/.caas/inventory/extra_groups‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,7 @@ cluster
77
[zenith:children]
88
grafana
99
openondemand
10+
11+
[manila:children]
12+
login
13+
compute

‎environments/.caas/inventory/group_vars/all/cluster.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,7 @@ openondemand_servername_default: "{{ hostvars[groups['openstack'][0]].cluster_ga
2020
openondemand_servername: "{{ zenith_fqdn_ood | default(openondemand_servername_default) }}"
2121

2222
appliances_state_dir: /var/lib/state
23+
24+
# Defaults for caas-provided extravars:
25+
cluster_project_manila_share: false
26+
cluster_home_manila_share: "{{ cluster_project_manila_share }}"

‎environments/.caas/inventory/group_vars/all/manila.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
caas_manila_home:
2+
share_name: "{{ cluster_name }}-home"
3+
mount_path: /home
4+
mount_user: root
5+
mount_group: root
6+
mount_mode: u=rwX,go=rX
7+
8+
cluster_project_manila_share_name: azimuth-project-share
9+
caas_manila_project:
10+
share_name: "{{ cluster_project_manila_share_name }}"
11+
mount_path: /project
12+
mount_user: root
13+
mount_group: root
14+
mount_mode: ugo=rwX
15+
16+
os_manila_mount_shares: "{{ ([caas_manila_home] if cluster_home_manila_share | bool else []) + ([caas_manila_project] if cluster_project_manila_share | bool else []) }}"

‎environments/.caas/inventory/group_vars/all/nfs.yml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
nfs_server: "{{ nfs_server_default }}"
22

3-
nfs_configurations:
4-
- comment: Export /exports/home from Slurm control node as /home
5-
nfs_enable:
6-
server: "{{ inventory_hostname in groups['control'] }}"
7-
clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
8-
nfs_export: "/exports/home" # assumes skeleton TF is being used
9-
nfs_client_mnt_point: "/home"
3+
caas_nfs_ood_state:
104
- comment: Export /var/lib/state from Slurm control node to OOD
115
nfs_enable:
126
server: "{{ inventory_hostname in groups['control'] }}"
137
clients: "{{ inventory_hostname in groups['openondemand'] }}"
148
nfs_export: "{{ appliances_state_dir }}"
159
nfs_client_mnt_point: "{{ appliances_state_dir }}"
1610
nfs_client_mnt_options: "x-systemd.required-by=zenith-ood.service,x-systemd.before=zenith-ood.service"
11+
12+
caas_nfs_home:
13+
- comment: Export /exports/home from Slurm control node as /home
14+
nfs_enable:
15+
server: "{{ inventory_hostname in groups['control'] }}"
16+
clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
17+
nfs_export: "/exports/home" # assumes skeleton TF is being used
18+
nfs_client_mnt_point: "/home"
19+
20+
nfs_configurations: "{{ caas_nfs_ood_state + (caas_nfs_home if not cluster_home_manila_share | bool else []) }}"

‎environments/.caas/ui-meta/slurm-infra.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ parameters:
5050

5151
- name: home_volume_size
5252
label: Home volume size (GB)
53-
description: The size of the cloud volume to use for home directories.
53+
description: The size of the cloud volume or share to use for home directories.
5454
kind: cloud.volume_size
5555
immutable: true
5656
options:

‎environments/.stackhpc/inventory/extra_groups‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,8 @@ cluster
2222
# [resolv_conf:children]
2323
# freeipa_client
2424
# --- end of FreeIPA example ---
25+
26+
[manila:children]
27+
# Allows demo; also installs manila client in fat image
28+
login
29+
compute

‎environments/.stackhpc/terraform/main.tf‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
# This terraform configuration uses the "skeleton" terraform, so that is checked by CI.
22

3+
terraform {
4+
required_version = ">= 0.14"
5+
required_providers {
6+
openstack = {
7+
source = "terraform-provider-openstack/openstack"
8+
}
9+
}
10+
}
11+
312
variable "environment_root" {
413
type = string
514
description = "Path to environment root, automatically set by activate script"
@@ -13,7 +22,7 @@ variable "cluster_name" {
1322
variable "cluster_image" {
1423
description = "single image for all cluster nodes - a convenience for CI"
1524
type = string
16-
default = "openhpc-240116-1156-aa8dba7d" # https://github.com/stackhpc/ansible-slurm-appliance/pull/351
25+
default = "openhpc-240116-1604-b3563a08" # https://github.com/stackhpc/ansible-slurm-appliance/pull/344
1726
# default = "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2"
1827
}
1928

0 commit comments

Comments
 (0)