Merge branch 'main' into fix/skelton-ssh-persist

Author: sjpb

.github/workflows/fatimage.yml

@@ -4,26 +4,26 @@ on:
   workflow_dispatch:
 jobs:
   openstack:
-    name: openstack-build-arcus
+    name: openstack-imagebuild
     concurrency: ${{ github.ref }} # to branch/PR
     runs-on: ubuntu-20.04
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
+      CI_CLOUD: ${{ vars.CI_CLOUD }}
     steps:
       - uses: actions/checkout@v2
 
       - name: Setup ssh
         run: |
           set -x
           mkdir ~/.ssh
-          echo "${arcus_SSH_KEY}" > ~/.ssh/id_rsa
+          echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
-        env:
-          arcus_SSH_KEY: ${{ secrets.ARCUS_SSH_KEY }}
+        shell: bash
 
       - name: Add bastion's ssh key to known_hosts
-        run: cat environments/.stackhpc/bastion_fingerprint >> ~/.ssh/known_hosts
+        run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
 
       - name: Install ansible etc
@@ -32,11 +32,9 @@ jobs:
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
-          echo "${arcus_CLOUDS_YAML}" > ~/.config/openstack/clouds.yaml
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
         shell: bash
-        env:
-          arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
-            
+
       - name: Setup environment
         run: |
           . venv/bin/activate
@@ -49,7 +47,7 @@ jobs:
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=ask -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
 
       - name: Get created image name from manifest
         id: manifest

.github/workflows/stackhpc.yml

@@ -40,6 +40,8 @@ jobs:
 
       - name: Install terraform
         uses: hashicorp/setup-terraform@v1
+        with:
+          terraform: v1.5.5
 
       - name: Initialise terraform
         run: terraform init

ansible/fatimage.yml

@@ -59,6 +59,11 @@
         tasks_from: vnc_compute.yml
 
     # - import_playbook: monitoring.yml:
+    - import_role:
+        name: opensearch
+        tasks_from: install.yml
+      become: true
+
     #   opensearch - containerised, nothing to do
     # slurm_stats - nothing to do
     # filebeat - containerised - nothing to do

ansible/monitoring.yml

@@ -7,6 +7,11 @@
   tasks:
     - import_role:
         name: opensearch
+        tasks_from: install.yml
+      become: true
+    - import_role:
+        name: opensearch
+        tasks_from: runtime.yml
       become: true
 
 - name: Setup slurm stats

ansible/roles/opensearch/defaults/main.yml

@@ -3,7 +3,7 @@
 #opensearch_internal_users_path:
 
 opensearch_podman_user: "{{ ansible_user }}"
-opensearch_version: '2.4.0' # https://hub.docker.com/r/opensearchproject/opensearch/tags
+opensearch_version: '2.9.0' # https://hub.docker.com/r/opensearchproject/opensearch/tags
 opensearch_config_path: /usr/share/opensearch/config
 opensearch_data_path: /usr/share/opensearch/data
 opensearch_state: started # will be restarted if required

ansible/roles/opensearch/handlers/main.yml

@@ -5,5 +5,4 @@
     name: opensearch.service
     state: "{{ 'restarted' if 'started' in opensearch_state else opensearch_state }}"
     enabled: "{{ opensearch_systemd_service_enabled }}"
-    daemon_reload: "{{ 'started' in opensearch_state }}"
   become: true

ansible/roles/opensearch/tasks/archive_data.yml

@@ -0,0 +1,17 @@
+# Remove data which was NOT indexed by Slurm Job ID
+# It will be re-ingested by filebeat from the slurmdbd, with that index
+
+- name: Ensure opensearch stopped
+  systemd:
+    name: opensearch
+    state: stopped
+  register: _opensearch_stop
+  until: "_opensearch_stop.status.ActiveState in ['inactive', 'failed']"
+  retries: 15
+  delay: 5
+
+- name: Archive existing data
+  community.general.archive:
+    path: "{{ opensearch_data_path }}"
+    dest: "{{ opensearch_data_path | dirname }}/data-{{ lookup('pipe', 'date --iso-8601=minutes') }}.tar.gz"
+    remove: true

ansible/roles/opensearch/tasks/install.yml

@@ -0,0 +1,19 @@
+# safe to use during build
+
+- name: Increase maximum number of virtual memory maps
+  # see https://opensearch.org/docs/2.0/opensearch/install/important-settings/
+  ansible.posix.sysctl:
+    name: vm.max_map_count
+    value: '262144'
+    state: present
+    reload: yes
+
+- name: Create systemd unit file
+  template:
+    dest: /etc/systemd/system/opensearch.service
+    src: opensearch.service.j2
+  register: _opensearch_unit
+
+- name: Reload opensearch unit file
+  command: systemctl daemon-reload
+  when: _opensearch_unit.changed

ansible/roles/opensearch/tasks/main.yml renamed to ansible/roles/opensearch/tasks/runtime.yml

@@ -15,14 +15,16 @@
     path: /etc/systemd/system/opendistro.service
     state: absent
 
-- name: Increase maximum number of virtual memory maps
-  # see https://opensearch.org/docs/2.0/opensearch/install/important-settings/
-  ansible.posix.sysctl:
-    name: vm.max_map_count
-    value: '262144'
-    state: present
-    reload: yes
-  become: true
+- name: Enumerate files in data directory
+  find:
+    path: "{{ opensearch_data_path }}"
+  register: _find_opensearch_data
+
+- name: Archive incorrectly indexed data
+  import_tasks: archive_data.yml
+  when:
+    - _find_opensearch_data.files | length > 0
+    - "'slurm_jobid_index' not in _find_opensearch_data.files | map(attribute='path') | map('basename')"
 
 - name: Ensure required opensearch host directories exist
   file:
@@ -35,11 +37,18 @@
   loop:
     - "{{ opensearch_config_path }}"
     - "{{ opensearch_data_path }}"
-  when: "'started' in opensearch_state" # don't run during image build
+
+- name: Set indexed data flag
+  copy:
+    dest: "{{ opensearch_data_path }}/slurm_jobid_index"
+    content: |
+      This is a flag file to indicate that filebeat is pushing data
+      indexed by Slurm JobID to prevent duplicate OpenSearch records
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
 
 - name: Create certs
   import_tasks: certs.yml
-  when: "'started' in opensearch_state" # don't run during image build
 
 - name: Template general configuration
   ansible.builtin.template:
@@ -52,7 +61,6 @@
     mode: 0660
   notify: Restart opensearch service
   become: true
-  when: "'started' in opensearch_state" # don't run during image build
 
 - name: Template internal user configuration
   template:
@@ -65,14 +73,11 @@
       mode: 0660
   notify: Restart opensearch service
   become: true
-  when: "'started' in opensearch_state" # don't run during image build
 
-- name: Create systemd unit file
-  template:
-    dest: /etc/systemd/system/opensearch.service
-    src: opensearch.service.j2
-  become: true
-  notify: Restart opensearch service
+- name: Pull container
+  containers.podman.podman_image:
+    name: "opensearchproject/opensearch:{{ opensearch_version }}"
+  become_user: "{{ opensearch_podman_user }}"
 
 - name: Flush handlers
   meta: flush_handlers

environments/.stackhpc/builder.pkrvars.hcl renamed to environments/.stackhpc/ARCUS.pkrvars.hcl

@@ -1,6 +1,6 @@
 flavor = "vm.ska.cpu.general.small"
 networks = ["a262aabd-e6bf-4440-a155-13dbc1b5db0e"] # WCDC-iLab-60
-source_image_name = "openhpc-230503-0944-bf8c3f63.qcow2" # https://github.com/stackhpc/ansible-slurm-appliance/pull/252
+source_image_name = "openhpc-230804-1754-80b8d714" # https://github.com/stackhpc/ansible-slurm-appliance/pull/298
 fatimage_source_image_name = "Rocky-8-GenericCloud-8.6.20220702.0.x86_64.qcow2"
 ssh_keypair_name = "slurm-app-ci"
 ssh_private_key_file = "~/.ssh/id_rsa"