Merge branch 'feature/k3s-monitoring' into feat/loki

Author: wtripp180901

.github/workflows/fatimage.yml

@@ -20,29 +20,23 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false # allow other matrix jobs to continue even if one fails
-      matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
+      matrix: # build RL8, RL9
         os_version:
           - RL8
           - RL9
         build:
           - openstack.openhpc
-          - openstack.openhpc-cuda
-        exclude:
-          - os_version: RL8
-            build: openstack.openhpc-cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
       SOURCE_IMAGES_MAP: |
         {
           "RL8": {
-            "openstack.openhpc": "rocky-latest-RL8",
-            "openstack.openhpc-cuda": "rocky-latest-cuda-RL8"
+            "openstack.openhpc": "rocky-latest-RL8"
           },
           "RL9": {
-            "openstack.openhpc": "rocky-latest-RL9",
-            "openstack.openhpc-cuda": "rocky-latest-cuda-RL9"
+            "openstack.openhpc": "rocky-latest-RL9"
           }
         }
 
@@ -117,4 +111,4 @@ jobs:
           path: |
             ./image-id.txt
             ./image-name.txt
-          overwrite: true
+          overwrite: true

.github/workflows/nightlybuild.yml

@@ -22,17 +22,12 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false # allow other matrix jobs to continue even if one fails
-      matrix: # build RL8, RL9, RL9+CUDA versions
+      matrix: # build RL8, RL9
         os_version:
           - RL8
           - RL9
         build:
           - openstack.rocky-latest
-          - openstack.rocky-latest-cuda
-        exclude:
-          - os_version: RL8
-            build: openstack.rocky-latest-cuda
-
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
@@ -108,68 +103,12 @@ jobs:
           echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
           echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
 
-      - name: Download image
+      - name: Make image usable for further builds
         run: |
           . venv/bin/activate
-          sudo mkdir /mnt/images
-          sudo chmod 777 /mnt/images
           openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
-          openstack image save --file /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-id }}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: install libguestfs
-        run: |
-          sudo apt -y update
-          sudo apt -y install libguestfs-tools
-
-      - name: mkdir for mount
-        run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
-
-      - name: mount qcow2 file
-        run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/[email protected]
-        with:
-          scan-type: fs
-          scan-ref: "${{ steps.manifest.outputs.image-name }}"
-          scanners: "vuln"
-          format: sarif
-          output: "${{ steps.manifest.outputs.image-name }}.sarif"
-          # turn off secret scanning to speed things up
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
-          category: "${{ matrix.os_version }}-${{ matrix.build }}"
-
-      - name: Fail if scan has CRITICAL vulnerabilities
-        uses: aquasecurity/[email protected]
-        with:
-          scan-type: fs
-          scan-ref: "${{ steps.manifest.outputs.image-name }}"
-          scanners: "vuln"
-          format: table
-          exit-code: '1'
-          severity: 'CRITICAL'
-          ignore-unfixed: true
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Delete new image if Trivy scan fails
-        if: failure() && steps.packer_build.outcome == 'success' # Runs if the Trivy scan found crit vulnerabilities or failed
-        run: |
-          . venv/bin/activate
-          echo "Deleting new image due to critical vulnerabilities or scan failure ..."
-          openstack image delete "${{ steps.manifest.outputs.image-id }}"
 
       - name: Delete old latest image
-        if: success() # Runs only if Trivy scan passed
         run: |
           . venv/bin/activate
           IMAGE_COUNT=$(openstack image list --name ${{ steps.manifest.outputs.image-name }} -f value -c ID | wc -l)
@@ -200,10 +139,7 @@ jobs:
           - RL9
         image:
           - rocky-latest
-          - rocky-latest-cuda
         exclude:
-          - os_version: RL8
-            image: rocky-latest-cuda
           - target_cloud: LEAFCLOUD
     env:
       OS_CLOUD: openstack

.github/workflows/s3-image-sync.yml

@@ -42,7 +42,6 @@ jobs:
         build:
           - RL8
           - RL9
-          - RL9-cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
@@ -112,7 +111,6 @@ jobs:
         build:
           - RL8
           - RL9
-          - RL9-cuda
         exclude: 
           - cloud: ${{ needs.image_upload.outputs.ci_cloud }}
 

.github/workflows/trivyscan.yml

@@ -10,13 +10,13 @@ on:
 jobs:
   scan:
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} # to branch/PR + OS + build
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} # to branch/PR + build
       cancel-in-progress: true
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        build: ["RL8", "RL9", "RL9-cuda"]
+        build: ["RL8", "RL9"]
     env:
       JSON_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
       OS_CLOUD: openstack
@@ -94,12 +94,13 @@ jobs:
           timeout: 15m
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
-          category: "${{ matrix.os_version }}-${{ matrix.build }}"
+          category: "${{ matrix.build }}"
 
       - name: Fail if scan has CRITICAL vulnerabilities
         uses: aquasecurity/[email protected]
@@ -114,3 +115,4 @@ jobs:
           timeout: 15m
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2

README.md

@@ -55,21 +55,28 @@ You will also need to install [OpenTofu](https://opentofu.org/docs/intro/install
 
 ### Create a new environment
 
-Use the `cookiecutter` template to create a new environment to hold your configuration. In the repository root run:
+Run the following from the repository root to activate the venv:
 
     . venv/bin/activate
+
+Use the `cookiecutter` template to create a new environment to hold your configuration:
+
     cd environments
     cookiecutter skeleton
 
 and follow the prompts to complete the environment name and description.
 
 **NB:** In subsequent sections this new environment is refered to as `$ENV`.
 
-Now generate secrets for this environment:
+Activate the new environment:
+
+    . environments/$ENV/activate
+
+And generate secrets for it:
 
     ansible-playbook ansible/adhoc/generate-passwords.yml
 
-### Define infrastructure configuration
+### Define and deploy infrastructure
 
 Create an OpenTofu variables file to define the required infrastructure, e.g.:
 
@@ -91,20 +98,28 @@ Create an OpenTofu variables file to define the required infrastructure, e.g.:
         }
     }
 
-Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables
-and descriptions see `environments/$ENV/terraform/terraform.tfvars`.
+Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables and descriptions see `environments/$ENV/terraform/terraform.tfvars`.
+
+To deploy this infrastructure, ensure the venv and the environment are [activated](#create-a-new-environment) and run:
 
-### Deploy appliance
+    export OS_CLOUD=openstack
+    cd environments/$ENV/terraform/
+    tofu apply
+
+and follow the prompts. Note the OS_CLOUD environment variable assumes that OpenStack credentials are defined using a [clouds.yaml](https://docs.openstack.org/python-openstackclient/latest/configuration/index.html#clouds-yaml) file in a default location with the default cloud name of `openstack`.
+
+### Configure appliance
+
+To configure the appliance, ensure the venv and the environment are [activated](#create-a-new-environment) and run:
 
     ansible-playbook ansible/site.yml
 
-You can now log in to the cluster using:
+Once it completes you can log in to the cluster using:
 
     ssh rocky@$login_ip
 
 where the IP of the login node is given in `environments/$ENV/inventory/hosts.yml`
 
-
 ## Overview of directory structure
 
 - `environments/`: See [docs/environments.md](docs/environments.md).

ansible/.gitignore

@@ -64,3 +64,5 @@ roles/*
 !roles/k9s/**
 !roles/kube_prometheus_stack
 !roles/kube_prometheus_stack/**
+!roles/lustre/
+!roles/lustre/**

ansible/bootstrap.yml

@@ -266,3 +266,4 @@
   tasks:
     - ansible.builtin.include_role:
         name: k3s
+        tasks_from: install.yml

ansible/cleanup.yml

@@ -38,11 +38,6 @@
 
 - name: Cleanup /tmp
   command : rm -rf /tmp/*
-
-- name: Delete ansible-init sentinel file created if ansible-init has run during build
-  ansible.builtin.file:
-    path: /var/lib/ansible-init.done
-    state: absent
 
 - name: Get package facts
   package_facts:

ansible/fatimage.yml

@@ -25,7 +25,7 @@
 
 - hosts: builder
   become: yes
-  gather_facts: no
+  gather_facts: yes
   tasks:
     # - import_playbook: iam.yml
     - name: Install FreeIPA client
@@ -44,6 +44,11 @@
         name: stackhpc.os-manila-mount
         tasks_from: install.yml
       when: "'manila' in group_names"
+    - name: Install Lustre packages
+      include_role:
+        name: lustre
+        tasks_from: install.yml
+      when: "'lustre' in group_names"
 
 - import_playbook: extras.yml
 
@@ -57,6 +62,7 @@
         name: mysql
         tasks_from: install.yml
       when: "'mysql' in group_names"
+
     - name: OpenHPC
       import_role:
         name: stackhpc.openhpc
@@ -83,18 +89,21 @@
       import_role:
         name: openondemand
         tasks_from: vnc_compute.yml
+
       when: "'openondemand_desktop' in group_names"
+
     - name: Open Ondemand jupyter node
       import_role:
         name: openondemand
         tasks_from: jupyter_compute.yml
-      when: "'openondemand' in group_names"
+      when: "'openondemand_jupyter' in group_names"
 
     # - import_playbook: monitoring.yml:
     - import_role:
         name: opensearch
         tasks_from: install.yml
       when: "'opensearch' in group_names"
+
     # slurm_stats - nothing to do
     - import_role:
         name: filebeat
@@ -114,6 +123,10 @@
         slurm_exporter_state: stopped
       when: "'slurm_exporter' in group_names"
 
+- hosts: prometheus
+  become: yes
+  gather_facts: yes
+  tasks:
     - name: kube prometheus stack
       import_role:
         name: kube_prometheus_stack

ansible/filesystems.yml

@@ -24,3 +24,13 @@
   tasks:
     - include_role:
         name: stackhpc.os-manila-mount
+
+- name: Setup Lustre clients
+  hosts: lustre
+  become: true
+  tags: lustre
+  tasks:
+    - include_role:
+        name: lustre
+        # NB install is ONLY run in builder
+        tasks_from: configure.yml