Merge branch 'main' into feat/compute-vols

Author: sjpb

.github/workflows/nightly-cleanup.yml

@@ -0,0 +1,71 @@
+name: Cleanup CI clusters
+on:
+  workflow_dispatch:
+      inputs:
+        ci_cloud:
+          description: 'Select the CI_CLOUD'
+          required: true
+          type: choice
+          options:
+            - LEAFCLOUD
+            - SMS
+            - ARCUS
+  schedule:
+    - cron: '0 20 * * *'  # Run at 8PM - image sync runs at midnight
+
+jobs:
+  ci_cleanup:
+    name: ci-cleanup
+    concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }}
+    strategy:
+      fail-fast: false
+      matrix:
+        cloud:
+          - LEAFCLOUD
+          - SMS
+          - ARCUS
+    runs-on: ubuntu-22.04
+    env:
+      OS_CLOUD: openstack
+      CI_CLOUD: ${{ matrix.cloud }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Record which cloud CI is running on
+        run: |
+          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+
+      - name: Setup environment
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          pip install -U pip
+          pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
+        shell: bash
+
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+        shell: bash
+
+      - name: Find CI clusters
+        run: |
+          . venv/bin/activate
+          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq)
+          echo "ci_clusters=${CI_CLUSTERS}" >> GITHUB_ENV
+        shell: bash
+      
+      - name: Delete clusters if control node not tagged with keep
+        run: |
+          . venv/bin/activate
+          for cluster_prefix in ${CI_CLUSTERS}
+          do
+            TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
+            if [[ $TAGS =~ "keep" ]]; then
+              echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+            else
+              yes | ./dev/delete-cluster.py ${cluster_prefix}
+            fi
+          done
+        shell: bash

.github/workflows/nightlybuild.yml

@@ -89,7 +89,7 @@ jobs:
           -on-error=${{ vars.PACKER_ON_ERROR }} \
           -only=${{ matrix.build }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-          -var "source_image_name=${{ env.SOURCE_IMAGE }}"
+          -var "source_image_name=${{ env.SOURCE_IMAGE }}" \
           openstack.pkr.hcl
 
         env:

.github/workflows/stackhpc.yml

@@ -12,6 +12,8 @@ on:
       - '!docs/**'
       - '!README.md'
       - '!.gitignore'
+      - '!.github/workflows/'
+      - '.github/workflows/stackhpc'
   pull_request:
     paths:
       - '**'
@@ -20,6 +22,8 @@ on:
       - '!docs/**'
       - '!README.md'
       - '!.gitignore'
+      - '!.github/workflows/'
+      - '.github/workflows/stackhpc'
 jobs:
   openstack:
     name: openstack-ci

ansible/roles/cluster_infra/tasks/main.yml

@@ -10,20 +10,20 @@
 # outputs.cluster_gateway_ip.
 - block:
     - name: Look up floating IP
-      include_role:
-        name: stackhpc.terraform.infra
-        tasks_from: lookup_floating_ip
-      vars:
-        os_floating_ip_id: "{{ cluster_floating_ip }}"
+      azimuth_cloud.terraform.os_floating_ip_info:
+         floating_ip: "{{ cluster_floating_ip }}"
+      register: cluster_floating_ip_info
 
     - name: Set floating IP address fact
       set_fact:
-        cluster_floating_ip_address: "{{ os_floating_ip_info.floating_ip_address }}"
-  when: cluster_floating_ip is defined
+        cluster_floating_ip_address: "{{ cluster_floating_ip_info.floating_ip.floating_ip_address }}"
+  when:
+    - cluster_floating_ip is defined
+    - cluster_floating_ip
 
 - name: Install Terraform binary
   include_role:
-    name: stackhpc.terraform.install
+    name: azimuth_cloud.terraform.install
 
 - name: Make Terraform project directory
   file:
@@ -59,4 +59,4 @@
 
 - name: Provision infrastructure
   include_role:
-    name: stackhpc.terraform.infra
+    name: azimuth_cloud.terraform.infra

ansible/roles/podman/defaults/main.yml

@@ -1,3 +1,2 @@
 podman_users:
   - name: "{{ ansible_user }}"
-podman_tmp_dir_root: /run # MUST be on a tmpfs

ansible/roles/podman/tasks/config.yml

@@ -44,3 +44,19 @@
   with_items: "{{ podman_users }}"
   register: podman_user_info
   become: yes
+
+- name: Clear up podman temporary files on startup
+  copy:
+    content: |
+      # Created by ansible
+      # Delete ephemeral podman files to avoid issues where /tmp is not of type tmpfs and persists across reboots.
+      # See tmpfiles.d(5) man page, note "R" specifies recursive removal and "!" marks as happening only on boot.
+      #
+      # Type Path Mode User Group Age Argument
+      R! /tmp/containers-user-*
+      R! /tmp/podman-run-*
+    dest: /etc/tmpfiles.d/podman-local.conf
+    owner: root
+    group: root
+    mode: 0660
+  become: true

ansible/roles/podman/tasks/validate.yml

@@ -29,16 +29,6 @@
           'enable_configless' not found in openhpc_config.SlurmctldParameters - is variable openhpc_config overridden?
           Additional slurm.conf parameters should be provided using variable openhpc_config_extra.
         success_msg: Checked Slurm will be configured for configless operation
-    
-- name: Validate podman configuration
-  hosts: podman
-  gather_facts: false
-  tags: podman
-  tasks:
-    - import_role:
-        name: podman
-        tasks_from: validate.yml
-      tags: validate
 
 - name: Validate filebeat configuration
   hosts: filebeat

ansible/validate.yml

@@ -10,5 +10,7 @@
         state: absent
       with_items:
         - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.7-1/gems/bootstrap_form-2.7.0/test/dummy/Gemfile.lock
+        - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.9-1/gems/bootstrap_form-2.7.0/test/dummy/Gemfile.lock
         - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.7-1/gems/bootstrap_form-4.5.0/demo/yarn.lock
+        - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.9-1/gems/bootstrap_form-4.5.0/demo/yarn.lock
         - /var/www/ood/apps/sys/dashboard/node_modules/data-confirm-modal/Gemfile.lock

environments/.stackhpc/hooks/post.yml

@@ -1,7 +1,7 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-241009-1523-354b048a",
-        "RL9": "openhpc-RL9-241009-1523-354b048a",
-        "RL9-cuda": "openhpc-cuda-RL9-241009-1523-354b048a"
+        "RL8": "openhpc-RL8-241022-0441-a5affa58",
+        "RL9": "openhpc-RL9-241022-0038-a5affa58",
+        "RL9-cuda": "openhpc-cuda-RL9-241022-0441-a5affa58"
     }
 }