Support clusters with no outbound internet

ansible/.gitignore

@@ -94,3 +94,5 @@ roles/*
 !roles/slurm_recompile/**
 !roles/nhc/
 !roles/nhc/**
+!roles/eessi/
+!roles/eessi/**

ansible/bootstrap.yml

@@ -186,8 +186,9 @@
   become: yes
   tasks:
     - name: Install and configure tuneD
-      import_role:
+      include_role:
         name: tuned
+        tasks_from: "{{ 'configure.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - hosts: freeipa_server
   # Done here as it might be providing DNS
@@ -217,31 +218,27 @@
   become: yes
   tags: firewalld
   tasks:
-    - import_role:
+    - include_role:
         name: firewalld
+        tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - hosts: fail2ban
   gather_facts: false
   become: yes
   tags: fail2ban
   tasks:
-    - import_role:
+    - include_role:
         name: fail2ban
+        tasks_from: "{{ 'configure.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - name: Setup podman
   gather_facts: false
   hosts: podman
   tags: podman
   tasks:
-    - import_role:
-        name: podman
-        tasks_from: prereqs.yml
-      tags: prereqs
-
-    - import_role:
+    - include_role:
         name: podman
-        tasks_from: config.yml
-      tags: config
+        tasks_from: "{{ 'configure.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - hosts: update
   gather_facts: false
@@ -317,8 +314,10 @@
   become: yes
   tags: linux_ansible_init
   tasks:
-    - include_role:
+    - name: Install ansible-init
+      include_role:
         name: azimuth_cloud.image_utils.linux_ansible_init
+      when: "appliances_mode == 'build'"
 
 - hosts: k3s:&builder
   become: yes

ansible/extras.yml

@@ -34,9 +34,10 @@
   become: true
   gather_facts: false
   tasks:
-    - name: Install and configure EESSI
-      import_role:
+    - name: Install / configure EESSI
+      include_role:
         name: eessi
+        tasks_from: "{{ 'configure.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - name: Setup CUDA
   hosts: cuda

ansible/fatimage.yml

@@ -108,7 +108,12 @@
         tasks_from: install.yml
       when: "'mysql' in group_names"
 
-    - name: OpenHPC
+    - name: Install rebuild
+      include_role:
+        name: rebuild
+        tasks_from: install.yml
+
+    - name: Install OpenHPC
       import_role:
         name: stackhpc.openhpc
         tasks_from: install.yml
@@ -134,7 +139,6 @@
       import_role:
         name: openondemand
         tasks_from: vnc_compute.yml
-
       when: "'openondemand_desktop' in group_names"
 
     - name: Open Ondemand jupyter node
@@ -153,7 +157,11 @@
         tasks_from: install.yml
       when: "'opensearch' in group_names"
 
-    # slurm_stats - nothing to do
+    - import_role:
+        name: slurm_stats
+        tasks_from: install.yml
+      when: "'slurm_stats' in group_names"
+
     - import_role:
         name: filebeat
         tasks_from: install.yml
@@ -171,11 +179,9 @@
       when: "'openondemand' in group_names"
 
     - name: slurm exporter
-      import_role:
+      include_role:
         name: slurm_exporter
-        tasks_from: install
-      vars:
-        slurm_exporter_state: stopped
+        tasks_from: install.yml
       when: "'slurm_exporter' in group_names"
 
     - name: Install alertmanager
@@ -249,6 +255,11 @@
     - import_role:
         name: cloudalchemy.grafana
         tasks_from: install.yml
+    - import_role:
+        name: cloudalchemy.grafana
+        tasks_from: plugins.yml
+    - include_role: # done in same play so it can use handlers from cloudalchemy.grafana
+        name: grafana-dashboards
 
 - name: Add support for NVIDIA GPU auto detection to Slurm
   hosts: cuda

ansible/filesystems.yml

@@ -24,6 +24,8 @@
   tasks:
     - include_role:
         name: stackhpc.os-manila-mount
+        tasks_from: "{{ item }}"
+      loop: "{{ ['lookup.yml', 'mount.yml'] if appliances_mode == 'configure' else ['main.yml'] }}"
 
 - name: Setup Lustre clients
   hosts: lustre

ansible/final.yml

@@ -17,3 +17,14 @@
     - include_role:
         name: compute_init
         tasks_from: export.yml
+
+- hosts: proxy
+  gather_facts: false
+  tags: proxy
+  become: yes
+  tasks:
+    - include_role:
+        name: proxy
+      vars:
+        proxy_state: absent
+      when: proxy_remove | default(false) | bool == true

ansible/iam.yml

@@ -20,9 +20,10 @@
   become: yes
   tasks:
     - name: Install FreeIPA client
-      import_role:
+      include_role:
         name: freeipa
         tasks_from: client-install.yml
+      when: "appliances_mode != 'configure'"
     - name: Enrol FreeIPA client
       import_role:
         name: freeipa

ansible/monitoring.yml

@@ -20,19 +20,22 @@
   tasks:
     - include_role:
         name: slurm_stats
+        tasks_from: "{{ 'configure.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - name: Deploy filebeat
   hosts: filebeat
   tags: filebeat
   tasks:
-    - import_role:
+    - include_role:
         name: filebeat
+        tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - name: Deploy node_exporter
   hosts: node_exporter
   tags: node_exporter
   tasks:
-    - import_role: name=cloudalchemy.node_exporter
+    - import_role:
+        name: cloudalchemy.node_exporter
 
 - name: Deploy OpenOndemand exporter
   hosts: openondemand
@@ -46,12 +49,13 @@
         tasks_from: exporter.yml
 
 - name: Deploy Slurm exporter
-  hosts: control
+  hosts: slurm_exporter
   become: true
   tags: slurm_exporter
   tasks:
-    - import_role:
+    - include_role:
         name: slurm_exporter
+        tasks_from: "{{ 'configure.yml' if appliances_mode == 'configure' else 'main.yml' }}"
 
 - name: Setup core monitoring software
   hosts: prometheus
@@ -68,24 +72,36 @@
       # i.e. if prometheus_version isn't defined we don't care, so use what's already there
       set_fact:
         prometheus_skip_install: "{{ false if prometheus_version is defined else true }}" 
-      when: "{{ (prometheus_binaries.results | map(attribute='stat') | map(attribute='exists')) + [prometheus_skip_install is not defined] }}"
+      when: "(prometheus_binaries.results | map(attribute='stat') | map(attribute='exists')) + [prometheus_skip_install is not defined]"
     - import_role:
         name: cloudalchemy.prometheus
 
 - name: Deploy grafana
   hosts: grafana
   tags: grafana
   tasks:
-    - assert:
-        that: vault_grafana_admin_password is defined
-        fail_msg: "Must define vault_grafana_admin_password - use `ansible-playbook generate-passwords.yml` to generate a set of passwords"
+    - name: Skip plugin installation in configure mode
+      # done during fatimage - can't do this in vars block as that is recursive
+      ansible.builtin.set_fact:
+        grafana_plugins: "{{ [] if appliances_mode == 'configure' else grafana_plugins }}"
+    - name: Copy Grafana plugins installed in image into persistent grafana state
+      ansible.builtin.copy:
+        remote_src: true
+        src: /var/lib/grafana/plugins/ # trailing / means copy contents
+        dest: "{{ grafana_data_dir }}/plugins/"
+        # below matches what already exists:
+        owner: root
+        group: root
+        mode: '0755'
+      become: true
     - include_role:
         name: cloudalchemy.grafana
       vars:
-        # We use internal roles to register the dashboards as the role does not support all options that we require.
+        # Internal role used to install dashboards as cloudalchemy role does not support all required options:
         grafana_dashboards: []
-    - import_role: # done in same play so it can use handlers from cloudalchemy.grafana
+    - include_role: # done in same play so it can use handlers from cloudalchemy.grafana
         name: grafana-dashboards
+      when: "appliances_mode != 'configure'"
 
 - name: Deploy alertmanager
   hosts: alertmanager

ansible/portal.yml

@@ -5,6 +5,10 @@
   become: yes
   gather_facts: yes # TODO
   tasks:
+    - name: Skip openondemand apps installation in configure mode
+      set_fact:
+        ood_install_apps: {}
+      when: appliances_mode == 'configure'
     - import_role:
         name: openondemand
         tasks_from: main.yml
@@ -19,6 +23,7 @@
     - import_role:
         name: openondemand
         tasks_from: vnc_compute.yml
+      when: appliances_mode != 'configure' # is run during build
 
 - hosts: openondemand_jupyter
   tags:
@@ -30,3 +35,4 @@
     - import_role:
         name: openondemand
         tasks_from: jupyter_compute.yml
+      when: appliances_mode != 'configure' # is run during build

ansible/roles/dnf_repos/tasks/disable_repos.yml

@@ -1,5 +1,5 @@
 ---
-- name: Disable Pulp repos
+- name: Remove password and disable Pulp repos
   ansible.builtin.yum_repository:
     file: "{{ item.file }}"
     name: "{{ item.name }}"
@@ -8,11 +8,25 @@
     enabled: false
   loop: "{{ dnf_repos_repolist }}"
 
-- name: Disable EPEL repo
+- name: Remove password and disable EPEL repo
   ansible.builtin.yum_repository:
     name: epel
     file: epel
     description: "{{ dnf_repos_epel_description }}"
     baseurl: "{{ dnf_repos_epel_baseurl }}"
     gpgcheck: false
     enabled: false
+
+- name: Get all repo files
+  ansible.builtin.find:
+    paths: /etc/yum.repos.d
+    patterns: '*.repo'
+  register: _dnf_repo_files
+
+- name: Disable every repo
+  ansible.builtin.replace:
+    path: "{{ item.path }}"
+    regexp: '^enabled\ ?=\ ?1'
+    replace: 'enabled=0'
+    backup: yes
+  loop: "{{ _dnf_repo_files.files }}"

ansible/roles/eessi/tasks/configure.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Add base CVMFS config
+  community.general.ini_file:
+    dest: /etc/cvmfs/default.local
+    section: null
+    option: "{{ item.key }}"
+    value: "{{ item.value }}"
+    no_extra_spaces: true
+  loop: "{{ cvmfs_config | dict2items }}"
+
+
+# NOTE: Not clear how to make this idempotent
+- name: Ensure CVMFS config is setup
+  command:
+    cmd: "cvmfs_config setup"

ansible/roles/eessi/tasks/main.yaml → ansible/roles/eessi/tasks/install.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: Download Cern GPG key
   ansible.builtin.get_url:
     url: http://cvmrepo.web.cern.ch/cvmrepo/yum/RPM-GPG-KEY-CernVM
@@ -31,18 +32,3 @@
 # - name: Install EESSI CVMFS config
 #   dnf:
 #     name: cvmfs-config-eessi
-
-- name: Add base CVMFS config
-  community.general.ini_file:
-    dest: /etc/cvmfs/default.local
-    section: null
-    option: "{{ item.key }}"
-    value: "{{ item.value }}"
-    no_extra_spaces: true
-  loop: "{{ cvmfs_config | dict2items }}"
-
-
-# NOTE: Not clear how to make this idempotent
-- name: Ensure CVMFS config is setup
-  command:
-    cmd: "cvmfs_config setup"

ansible/roles/eessi/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+
+- include_tasks: install.yml
+- include_tasks: configure.yml

ansible/roles/fail2ban/tasks/configure.yml

@@ -0,0 +1,15 @@
+---
+- name: Create config
+  template:
+    dest: /etc/fail2ban/jail.local
+    src: jail.local.j2
+  notify: Restart fail2ban
+
+- name: flush handlers
+  meta: flush_handlers
+
+- name: Ensure fail2ban running even if no config change
+  service:
+    name: fail2ban
+    state: started
+    enabled: true

ansible/roles/fail2ban/tasks/install.yml

@@ -0,0 +1,11 @@
+---
+- name: Install EPEL repo
+  package:
+    name: epel-release
+
+- name: Install fail2ban packages
+  package:
+    name:
+      - fail2ban-server
+      - fail2ban-firewalld
+    state: present