Merge pull request #3 from stackhpc/feature/ood

Add Open OnDemand to CaaS clusters

Author: Matt Pryor

assets/ood-icon.png

@@ -0,0 +1,6 @@
+basic_users_users:
+  - name: azimuth
+    # Hash the password with a salt that is different for each host
+    password: "{{ vault_azimuth_user_password | password_hash('sha512', 65534 | random(seed=inventory_hostname) | string) }}"
+    uid: 1005
+    public_key: "{{ cluster_user_ssh_public_key }}"

group_vars/basic_users.yml

@@ -2,6 +2,7 @@
 update_enable: "{{ cluster_upgrade_system_packages | default('false') | bool }}"
 
 # Read the secrets from the Ansible local facts on the control host
+vault_azimuth_user_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_azimuth_user_password }}"
 vault_grafana_admin_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_grafana_admin_password }}"
 vault_elasticsearch_admin_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_elasticsearch_admin_password }}"
 vault_elasticsearch_kibana_password: "{{ hostvars[groups['control'][0]].ansible_local.openhpc_secrets.vault_elasticsearch_kibana_password }}"
@@ -11,3 +12,7 @@ vault_openhpc_mungekey: "{{ hostvars[groups['control'][0]].ansible_local.openhpc
 
 # Override this to cope with the case where the podman group just doesn't exist
 appliances_local_users_podman_enable: "{{ groups.get('podman', []) | length > 0 }}"
+
+# The server name for Open OnDemand depends on whether Zenith is enabled or not
+openondemand_servername_default: "{{ hostvars[groups['openstack'][0]].cluster_floating_ip_address | replace('.', '-') ~ '.sslip.io' }}"
+openondemand_servername: "{{ zenith_fqdn_ood | default(openondemand_servername_default) }}"

group_vars/cluster.yml

@@ -1,6 +1,7 @@
 ---
 
-zenith_proxy_podman_user: podman
+# Override when there's no OOD
+grafana_serve_from_sub_path: "{{ 'openondemand' in groups }}"
+grafana_auth_anonymous: "{{ 'openondemand' in groups }}"
 
-grafana_address: "{{ ansible_default_ipv4.address }}"
-grafana_port: 3000
+grafana_url: "{{ grafana_url_openondemand_proxy if 'openondemand' in groups else grafana_url_direct }}"

group_vars/grafana.yml

@@ -1 +1,5 @@
 openhpc_cluster_name: "{{ cluster_name }}"
+
+openhpc_config:
+  SlurmctldDebug: debug
+  SlurmdDebug: debug

group_vars/openhpc.yml

@@ -0,0 +1,77 @@
+---
+openondemand_auth: basic_pam
+openondemand_jupyter_partition: "compute"
+openondemand_desktop_partition: "compute"
+
+httpd_listen_addr_port:
+  - 80
+  - 443
+
+# Allow proxying to compute nodes for apps and control for monitoring only when the grafana group is available
+openondemand_host_regex: '({{ openhpc_cluster_name ~ "-compute-\d+)" ~ ( "|(" ~ groups["grafana"][0] ~ ")" if "grafana" in groups else "" ) }}'
+
+# Add grafana to dashboard links to OOD only if grafana group is available
+openondemand_dashboard_links_grafana:
+  - name: Grafana
+    app_name: grafana
+    category: Monitoring
+    description: Dashboards
+    url: "{{ grafana_url_openondemand_proxy }}"
+openondemand_dashboard_links: "{{ openondemand_dashboard_links_grafana if 'grafana' in groups else [] }}"
+
+# Add grafana panel to jobs page only if grafana group is available
+openondemand_clusters:
+  slurm:
+    v2:
+      metadata:
+        title: "{{ openhpc_cluster_name }}" # interpolation here works as openondemand is lexically after openhpc
+      login:
+        host: "{{ hostvars[groups['login'].0].api_address }}"
+        default: true
+      job:
+        adapter: slurm
+        cluster: "{{ openhpc_cluster_name }}"
+      batch_connect:
+        basic:
+          script_wrapper: |-
+            module purge
+            export PATH=/opt/jupyter/bin/:$PATH
+            %s
+          set_host: host=$(hostname -s)
+        vnc:
+          script_wrapper: |-
+            module purge
+            export PATH=/opt/TurboVNC/bin:$PATH
+            # Workaround to avoid "Unable to contact settings server" when
+            # lauching xfce4-session
+            xfce4-session() { /bin/dbus-launch /bin/xfce4-session $@ ; }
+            export -f xfce4-session
+            %s
+          set_host: host=$(hostname -s)
+      custom: "{{ openondemand_clusters_grafana if 'grafana' in groups else {} }}"
+
+grafana_address: "{{ hostvars[groups['grafana'][0]]['api_address'] if 'grafana' in groups else '' }}"
+grafana_url_openondemand_proxy: "https://{{ openondemand_servername }}/node/{{ groups['grafana'][0] if 'grafana' in groups else '' }}/{{ grafana_port }}"
+
+openondemand_clusters_grafana: 
+  # embed grafana panels in Jobs app: https://osc.github.io/ood-documentation/latest/customization.html#grafana-support
+  grafana:
+    host: "{{ grafana_url_openondemand_proxy if 'openondemand' in groups else grafana_url_direct }}"
+    orgId: 1
+    dashboard:
+      name: "node-exporter-slurm"
+      uid: "node-exporter-slurm"
+      panels:
+        cpu: 77
+        memory: 78
+    labels:
+      cluster: "cluster"
+      host: "host"
+      jobid: "jobid"
+
+_opeonondemand_unset_auth: '    RequestHeader unset Authorization'
+
+# Fix grafana proxying for basic auth if anonymous grafana access enabled:
+openondemand_node_proxy_directives: "{{ _opeonondemand_unset_auth if (openondemand_auth == 'basic_pam' and 'openondemand_host_regex' and 'grafana' in groups and hostvars[groups['grafana'][0]]._grafana_auth_is_anonymous) else '' }}"
+
+

group_vars/openondemand.yml

@@ -1,49 +1,18 @@
 # The default Terraform state key for backends that support it
 terraform_state_key: "cluster/{{ cluster_id }}/tfstate"
 
+# Set up the terraform backend
+terraform_backend_type: "{{ 'consul' if 'CONSUL_HTTP_ADDR' in ansible_env else 'local' }}"
 terraform_backend_config_defaults:
   consul:
     path: "{{ terraform_state_key }}"
     gzip: "true"
   local: {}
+terraform_backend_config: "{{ terraform_backend_config_defaults[terraform_backend_type] }}"
 
-#####
-## WARNING
-##
-## The groups specified here should replicate the groups in the StackHPC Slurm appliance environments
-##
-## https://github.com/stackhpc/ansible-slurm-appliance/blob/main/environments/common/inventory/groups
-## https://github.com/stackhpc/ansible-slurm-appliance/blob/main/environments/common/layouts/everything
-## https://github.com/stackhpc/ansible-slurm-appliance/blob/main/environments/common/layouts/minimal
-#####
-# These groups should represent the minimal layout
-cluster_groups_required:
-  login:         ["{{ cluster_name }}_login"]
-  control:       ["{{ cluster_name }}_control"]
-  compute:       ["{{ cluster_name }}_compute"]
-  openhpc:       [login, control, compute]
-  cluster:       [openhpc]
-  selinux:       [cluster]
-  nfs:           [cluster]
-  mysql:         [control]
-  update:        [cluster]
+terraform_binary_directory: "{{ playbook_dir }}/bin"
+terraform_binary_path: "{{ terraform_binary_directory }}/terraform"
+terraform_project_path: "{{ playbook_dir }}/terraform"
 
-# These are the additional groups required for monitoring (see everything layout)
-cluster_groups_monitoring:
-  podman:        [opendistro, filebeat]
-  prometheus:    [control]
-  grafana:       [control]
-  alertmanager:  [control]
-  node_exporter: [cluster]
-  opendistro:    [control]
-  slurm_stats:   [control]
-  filebeat:      [slurm_stats]
-
-# Additional groups for running the cluster validation
-cluster_groups_validation:
-  hpctests: [login]
-
-# Additional groups for Zenith support
-cluster_groups_zenith:
-  # Any hosts in the grafana group should go in the zenith group
-  zenith: [grafana]
+terraform_state: "{{ cluster_state | default('present') }}"
+cluster_ssh_user: rocky

group_vars/openstack.yml

@@ -0,0 +1,8 @@
+---
+# Override openondemand_address because its needed in openondemand_scrape_configs
+# which is used in prometheus_scrape_configs
+openondemand_address: "{{ hostvars[groups['openondemand'].0].api_address if 'openondemand' in groups else '' }}"
+
+# Override group_var set in ansible-slurm-appliance all group - unless
+# OOD is being deployed then there won't be an OOD group
+prometheus_scrape_configs: "{{ prometheus_scrape_configs_default + (openondemand_scrape_configs if ( 'openondemand' in groups ) else [] ) }}"

group_vars/prometheus.yml

@@ -0,0 +1 @@
+zenith_proxy_podman_user: podman

group_vars/zenith.yml

@@ -18,10 +18,10 @@ roles:
   - src: cloudalchemy.grafana
   - src: geerlingguy.mysql
   - src: jriguera.configdrive
-  - name: stackhpc.terraform-infra
-    src: https://github.com/stackhpc/ansible-role-terraform-infra
+  - src: https://github.com/OSC/ood-ansible.git
+    name: osc.ood
     type: git
-    version: 4d9d67b5a1866edf6988a1f7e9e64868df8f65ae
+    version: v2.0.5
 
 collections:
   - name: ansible.posix
@@ -33,6 +33,9 @@ collections:
   - name: community.mysql
   - name: containers.podman
   - name: openstack.cloud
+  - name: https://github.com/stackhpc/ansible-collection-terraform
+    type: git
+    version: ae1dc46a9d266bcdc6e79a6e290edbb080596f7f
   - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools
     type: git
     version: v0.1.0