diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml
new file mode 100644
index 000000000..bf438c336
--- /dev/null
+++ b/.github/workflows/extra.yml
@@ -0,0 +1,139 @@
+name: Test extra build
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
+      - 'ansible/roles/doca/**'
+      - 'ansible/roles/cuda/**'
+      - 'ansible/roles/lustre/**'
+      - '.github/workflows/extra.yml'
+  pull_request:
+    paths:
+      - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
+      - 'ansible/roles/doca/**'
+      - 'ansible/roles/cuda/**'
+      - 'ansible/roles/lustre/**'
+      - '.github/workflows/extra.yml'
+
+jobs:
+  doca:
+    name: extra-build
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
+      cancel-in-progress: true
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false # allow other matrix jobs to continue even if one fails
+      matrix: # build RL8, RL9
+        build:
+          - image_name: openhpc-extra-RL8
+            source_image_name_key: RL8 # key into environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
+            inventory_groups: doca,cuda,lustre
+            volume_size: 30 # needed for cuda
+          - image_name: openhpc-extra-RL9
+            source_image_name_key: RL9
+            inventory_groups: doca,cuda,lustre
+            volume_size: 30 # needed for cuda
+    env:
+      ANSIBLE_FORCE_COLOR: True
+      OS_CLOUD: openstack
+      CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings
+      ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Load current fat images into GITHUB_ENV
+        # see https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#example-of-a-multiline-string
+        run: |
+          {
+            echo 'FAT_IMAGES<<EOF'
+            cat environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
+            echo EOF
+          } >> "$GITHUB_ENV"
+
+      - name: Record settings
+        run: |
+          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+          echo FAT_IMAGES: ${FAT_IMAGES}
+
+      - name: Setup ssh
+        run: |
+          set -x
+          mkdir ~/.ssh
+          echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+        shell: bash
+
+      - name: Add bastion's ssh key to known_hosts
+        run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
+        shell: bash
+
+      - name: Install ansible etc
+        run: dev/setup-env.sh
+
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+        shell: bash
+
+      - name: Setup environment
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+
+      - name: Build fat image with packer
+        id: packer_build
+        run: |
+          set -x
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          cd packer/
+          packer init .
+          
+          PACKER_LOG=1 packer build \
+          -on-error=${{ vars.PACKER_ON_ERROR }} \
+          -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
+          -var "source_image_name=${{ fromJSON(env.FAT_IMAGES)['cluster_image'][matrix.build.source_image_name_key] }}" \
+          -var "image_name=${{ matrix.build.image_name }}" \
+          -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
+          -var "volume_size=${{ matrix.build.volume_size }}" \
+          openstack.pkr.hcl
+        
+      - name: Get created image names from manifest
+        id: manifest
+        run: |
+          . venv/bin/activate
+          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
+          while ! openstack image show -f value -c name $IMAGE_ID; do
+            sleep 5
+          done
+          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
+          echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
+          echo $IMAGE_ID > image-id.txt
+          echo $IMAGE_NAME > image-name.txt
+
+      - name: Make image usable for further builds
+        run: |
+          . venv/bin/activate
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+
+      - name: Delete image for automatically-run workflows
+        run: |
+          . venv/bin/activate
+          openstack image delete "${{ steps.manifest.outputs.image-id }}"
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+
+      - name: Upload manifest artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-details-${{ matrix.build.image_name }}
+          path: |
+            ./image-id.txt
+            ./image-name.txt
+          overwrite: true
diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 5425eb4e3..c3b91fefa 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -15,36 +15,25 @@ jobs:
   openstack:
     name: openstack-imagebuild
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
       cancel-in-progress: true
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false # allow other matrix jobs to continue even if one fails
-      matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
-        os_version:
-          - RL8
-          - RL9
+      matrix: # build RL8, RL9
         build:
-          - openstack.openhpc
-          - openstack.openhpc-cuda
-        exclude:
-          - os_version: RL8
-            build: openstack.openhpc-cuda
+          - image_name: openhpc-RL8
+            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.qcow2
+            inventory_groups: control,compute,login,update
+          - image_name: openhpc-RL9
+            source_image_name: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2
+            inventory_groups: control,compute,login,update
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
-      SOURCE_IMAGES_MAP: |
-        {
-          "RL8": {
-            "openstack.openhpc": "rocky-latest-RL8",
-            "openstack.openhpc-cuda": "rocky-latest-cuda-RL8"
-          },
-          "RL9": {
-            "openstack.openhpc": "rocky-latest-RL9",
-            "openstack.openhpc-cuda": "rocky-latest-cuda-RL9"
-          }
-        }
+      ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
       - uses: actions/checkout@v2
@@ -90,13 +79,11 @@ jobs:
 
           PACKER_LOG=1 packer build \
           -on-error=${{ vars.PACKER_ON_ERROR }} \
-          -only=${{ matrix.build }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-          -var "source_image_name=${{ env.SOURCE_IMAGE }}" \
+          -var "source_image_name=${{ matrix.build.source_image_name }}" \
+          -var "image_name=${{ matrix.build.image_name }}" \
+          -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
           openstack.pkr.hcl
-        env:
-          PKR_VAR_os_version: ${{ matrix.os_version }}
-          SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version][matrix.build] }}
 
       - name: Get created image names from manifest
         id: manifest
@@ -107,14 +94,21 @@ jobs:
             sleep 5
           done
           IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
+          echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
           echo $IMAGE_ID > image-id.txt
           echo $IMAGE_NAME > image-name.txt
 
+      - name: Make image usable for further builds
+        run: |
+          . venv/bin/activate
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+
       - name: Upload manifest artifact
         uses: actions/upload-artifact@v4
         with:
-          name: image-details-${{ matrix.build }}-${{ matrix.os_version }}
+          name: image-details-${{ matrix.build.image_name }}
           path: |
             ./image-id.txt
             ./image-name.txt
-          overwrite: true
\ No newline at end of file
+          overwrite: true
diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml
index 9bfc5230a..0f7156fad 100644
--- a/.github/workflows/nightly-cleanup.yml
+++ b/.github/workflows/nightly-cleanup.yml
@@ -1,17 +1,8 @@
 name: Cleanup CI clusters
 on:
   workflow_dispatch:
-      inputs:
-        ci_cloud:
-          description: 'Select the CI_CLOUD'
-          required: true
-          type: choice
-          options:
-            - LEAFCLOUD
-            - SMS
-            - ARCUS
   schedule:
-    - cron: '0 20 * * *'  # Run at 8PM - image sync runs at midnight
+    - cron: '0 21 * * *'  # Run at 9PM - image sync runs at midnight
 
 jobs:
   ci_cleanup:
@@ -52,20 +43,55 @@ jobs:
       - name: Find CI clusters
         run: |
           . venv/bin/activate
-          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq)
-          echo "ci_clusters=${CI_CLUSTERS}" >> GITHUB_ENV
+          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq || true)
+          echo "DEBUG: Raw CI clusters: $CI_CLUSTERS"
+      
+          if [[ -z "$CI_CLUSTERS" ]]; then
+            echo "No matching CI clusters found."
+          else
+            # Flatten multiline value so can be passed as env var
+            CI_CLUSTERS_FORMATTED=$(echo "$CI_CLUSTERS" | tr '\n' ' ' | sed 's/ $//')
+            echo "DEBUG: Formatted CI clusters: $CI_CLUSTERS_FORMATTED"
+            echo "ci_clusters=$CI_CLUSTERS_FORMATTED" >> $GITHUB_ENV
+          fi
         shell: bash
       
       - name: Delete clusters if control node not tagged with keep
         run: |
           . venv/bin/activate
-          for cluster_prefix in ${CI_CLUSTERS}
+          if [[ -z ${ci_clusters} ]]; then
+            echo "No clusters to delete."
+            exit 0
+          fi
+          
+          for cluster_prefix in ${ci_clusters}
           do
-            TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
-            if [[ $TAGS =~ "keep" ]]; then
-              echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+            echo "Processing cluster: $cluster_prefix"
+            # Get all servers with the matching name for control node
+            CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
+            SERVER_COUNT=$(echo "$CONTROL_SERVERS" | jq length)
+
+            if [[ $SERVER_COUNT -gt 1 ]]; then
+              echo "Multiple servers found for control node '${cluster_prefix}-control'. Checking tags for each..."
+              
+              for server in $(echo "$CONTROL_SERVERS" | jq -r '.[].ID'); do
+                # Get tags for each control node
+                TAGS=$(openstack server show "$server" --column tags --format value)
+                
+                if [[ $TAGS =~ "keep" ]]; then
+                  echo "Skipping ${cluster_prefix} (server ${server}) - control instance is tagged as keep"
+                else
+                  ./dev/delete-cluster.py ${cluster_prefix} --force
+                fi
+              done
             else
-              yes | ./dev/delete-cluster.py ${cluster_prefix}
+              # If only one server, extract its tags and proceed
+              TAGS=$(echo "$CONTROL_SERVERS" | jq -r '.[0].Tags')
+              if [[ $TAGS =~ "keep" ]]; then
+                echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+              else
+                ./dev/delete-cluster.py ${cluster_prefix} --force
+              fi
             fi
           done
         shell: bash
diff --git a/.github/workflows/nightlybuild.yml b/.github/workflows/nightlybuild.yml
index 333550c53..ec920ce8d 100644
--- a/.github/workflows/nightlybuild.yml
+++ b/.github/workflows/nightlybuild.yml
@@ -11,37 +11,31 @@ on:
             - SMS
             - ARCUS
   schedule:
-    - cron: '0 0 * * *'  # Run at midnight
+    - cron: '0 0 * * *'  # Run at midnight on default branch
 
 jobs:
   openstack:
     name: openstack-imagebuild
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
       cancel-in-progress: true
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false # allow other matrix jobs to continue even if one fails
-      matrix: # build RL8, RL9, RL9+CUDA versions
-        os_version:
-          - RL8
-          - RL9
+      matrix: # build RL8, RL9
         build:
-          - openstack.rocky-latest
-          - openstack.rocky-latest-cuda
-        exclude:
-          - os_version: RL8
-            build: openstack.rocky-latest-cuda
-
+          - image_name: rocky-latest-RL8
+            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.qcow2
+            inventory_groups: update
+          - image_name: rocky-latest-RL9
+            source_image_name: Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2
+            inventory_groups: update
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }}
-      SOURCE_IMAGES_MAP: |
-        {
-          "RL8": "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2",
-          "RL9": "Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2"
-        }
+      ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
       - uses: actions/checkout@v2
@@ -87,15 +81,13 @@ jobs:
 
           PACKER_LOG=1 packer build \
           -on-error=${{ vars.PACKER_ON_ERROR }} \
-          -only=${{ matrix.build }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-          -var "source_image_name=${{ env.SOURCE_IMAGE }}" \
+          -var "source_image_name=${{ matrix.build.source_image_name }}" \
+          -var "image_name=${{ matrix.build.image_name }}" \
+          -var "image_name_version=" \
+          -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
           openstack.pkr.hcl
 
-        env:
-          PKR_VAR_os_version: ${{ matrix.os_version }}
-          SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version] }}
-
       - name: Get created image names from manifest
         id: manifest
         run: |
@@ -108,68 +100,12 @@ jobs:
           echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
           echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
 
-      - name: Download image
+      - name: Make image usable for further builds
         run: |
           . venv/bin/activate
-          sudo mkdir /mnt/images
-          sudo chmod 777 /mnt/images
           openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
-          openstack image save --file /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-id }}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: install libguestfs
-        run: |
-          sudo apt -y update
-          sudo apt -y install libguestfs-tools
-
-      - name: mkdir for mount
-        run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
-
-      - name: mount qcow2 file
-        run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.17.0
-        with:
-          scan-type: fs
-          scan-ref: "${{ steps.manifest.outputs.image-name }}"
-          scanners: "vuln"
-          format: sarif
-          output: "${{ steps.manifest.outputs.image-name }}.sarif"
-          # turn off secret scanning to speed things up
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
-          category: "${{ matrix.os_version }}-${{ matrix.build }}"
-
-      - name: Fail if scan has CRITICAL vulnerabilities
-        uses: aquasecurity/trivy-action@0.16.1
-        with:
-          scan-type: fs
-          scan-ref: "${{ steps.manifest.outputs.image-name }}"
-          scanners: "vuln"
-          format: table
-          exit-code: '1'
-          severity: 'CRITICAL'
-          ignore-unfixed: true
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Delete new image if Trivy scan fails
-        if: failure() && steps.packer_build.outcome == 'success' # Runs if the Trivy scan found crit vulnerabilities or failed
-        run: |
-          . venv/bin/activate
-          echo "Deleting new image due to critical vulnerabilities or scan failure ..."
-          openstack image delete "${{ steps.manifest.outputs.image-id }}"
 
       - name: Delete old latest image
-        if: success() # Runs only if Trivy scan passed
         run: |
           . venv/bin/activate
           IMAGE_COUNT=$(openstack image list --name ${{ steps.manifest.outputs.image-name }} -f value -c ID | wc -l)
@@ -185,7 +121,7 @@ jobs:
     name: upload-nightly-targets
     needs: openstack
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.image }}-${{ matrix.target_cloud }}
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }}-${{ matrix.target_cloud }}
       cancel-in-progress: true
     runs-on: ubuntu-22.04
     strategy:
@@ -195,21 +131,15 @@ jobs:
           - LEAFCLOUD
           - SMS
           - ARCUS
-        os_version:
-          - RL8
-          - RL9
-        image:
-          - rocky-latest
-          - rocky-latest-cuda
+        build:
+          - image_name: rocky-latest-RL8
+          - image_name: rocky-latest-RL9
         exclude:
-          - os_version: RL8
-            image: rocky-latest-cuda
           - target_cloud: LEAFCLOUD
     env:
       OS_CLOUD: openstack
       SOURCE_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }}
       TARGET_CLOUD: ${{ matrix.target_cloud }}
-      IMAGE_NAME: "${{ matrix.image }}-${{ matrix.os_version }}"
     steps:
       - uses: actions/checkout@v2
 
@@ -224,42 +154,37 @@ jobs:
           . venv/bin/activate
           pip install -U pip
           pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
-        shell: bash
 
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
           echo "${{ secrets[format('{0}_CLOUDS_YAML', env.SOURCE_CLOUD)] }}" > ~/.config/openstack/source_clouds.yaml
           echo "${{ secrets[format('{0}_CLOUDS_YAML', env.TARGET_CLOUD)] }}" > ~/.config/openstack/target_clouds.yaml
-        shell: bash
 
       - name: Download source image
         run: |
           . venv/bin/activate
           export OS_CLIENT_CONFIG_FILE=~/.config/openstack/source_clouds.yaml
-          openstack image save --file ${{ env.IMAGE_NAME }} ${{ env.IMAGE_NAME }}
-        shell: bash
+          openstack image save --file ${{ matrix.build.image_name }} ${{ matrix.build.image_name }}
 
       - name: Upload to target cloud
         run: |
           . venv/bin/activate
           export OS_CLIENT_CONFIG_FILE=~/.config/openstack/target_clouds.yaml
 
-          openstack image create "${{ env.IMAGE_NAME }}" \
-            --file "${{ env.IMAGE_NAME }}" \
+          openstack image create "${{ matrix.build.image_name }}" \
+            --file "${{ matrix.build.image_name }}" \
             --disk-format qcow2 \
-        shell: bash
 
       - name: Delete old latest image from target cloud
         run: |
           . venv/bin/activate
           export OS_CLIENT_CONFIG_FILE=~/.config/openstack/target_clouds.yaml
 
-          IMAGE_COUNT=$(openstack image list --name ${{ env.IMAGE_NAME }} -f value -c ID | wc -l)
+          IMAGE_COUNT=$(openstack image list --name ${{ matrix.build.image_name }} -f value -c ID | wc -l)
           if [ "$IMAGE_COUNT" -gt 1 ]; then
-            OLD_IMAGE_ID=$(openstack image list --sort created_at:asc --name "${{ env.IMAGE_NAME }}"  -f value -c ID | head -n 1)
+            OLD_IMAGE_ID=$(openstack image list --sort created_at:asc --name "${{ matrix.build.image_name }}"  -f value -c ID | head -n 1)
             openstack image delete "$OLD_IMAGE_ID"
           else
             echo "Only one image exists, skipping deletion."
           fi
-        shell: bash
diff --git a/.github/workflows/s3-image-sync.yml b/.github/workflows/s3-image-sync.yml
index 0ffaae954..85e0c2fad 100644
--- a/.github/workflows/s3-image-sync.yml
+++ b/.github/workflows/s3-image-sync.yml
@@ -42,7 +42,6 @@ jobs:
         build:
           - RL8
           - RL9
-          - RL9-cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
@@ -112,7 +111,6 @@ jobs:
         build:
           - RL8
           - RL9
-          - RL9-cuda
         exclude: 
           - cloud: ${{ needs.image_upload.outputs.ci_cloud }}
 
diff --git a/.github/workflows/trivyscan.yml b/.github/workflows/trivyscan.yml
index 2957b22ee..5b65baca1 100644
--- a/.github/workflows/trivyscan.yml
+++ b/.github/workflows/trivyscan.yml
@@ -10,13 +10,13 @@ on:
 jobs:
   scan:
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} # to branch/PR + OS + build
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} # to branch/PR + build
       cancel-in-progress: true
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        build: ["RL8", "RL9", "RL9-cuda"]
+        build: ["RL8", "RL9"]
     env:
       JSON_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
       OS_CLOUD: openstack
@@ -25,6 +25,20 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Override CI_CLOUD if PR label is present
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          # Iterate over the labels
+          labels=$(echo '${{ toJSON(github.event.pull_request.labels) }}' | jq -r '.[].name')
+          echo $labels
+          for label in $labels; do
+             if [[ $label == CI_CLOUD=* ]]; then
+              # Extract the value after 'CI_CLOUD='
+              CI_CLOUD_OVERRIDE=${label#CI_CLOUD=}
+              echo "CI_CLOUD=${CI_CLOUD_OVERRIDE}" >> $GITHUB_ENV
+            fi
+          done
+
       - name: Record settings for CI cloud
         run: |
           echo CI_CLOUD: ${{ env.CI_CLOUD }}
@@ -94,12 +108,13 @@ jobs:
           timeout: 15m
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
-          category: "${{ matrix.os_version }}-${{ matrix.build }}"
+          category: "${{ matrix.build }}"
 
       - name: Fail if scan has CRITICAL vulnerabilities
         uses: aquasecurity/trivy-action@0.24.0
@@ -114,3 +129,4 @@ jobs:
           timeout: 15m
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2
diff --git a/README.md b/README.md
index b54cd110a..54b74d799 100644
--- a/README.md
+++ b/README.md
@@ -6,10 +6,10 @@ This repository contains playbooks and configuration to define a Slurm-based HPC
 - [Rocky Linux](https://rockylinux.org/)-based hosts.
 - [OpenTofu](https://opentofu.org/) configurations to define the cluster's infrastructure-as-code.
 - Packages for Slurm and MPI software stacks from [OpenHPC](https://openhpc.community/).
-- Shared fileystem(s) using NFS (with in-cluster or external servers) or [CephFS](https://docs.ceph.com/en/latest/cephfs/) via [Openstack Manila](https://wiki.openstack.org/wiki/Manila).
+- Shared fileystem(s) using NFS (with in-cluster or external servers) or [CephFS](https://docs.ceph.com/en/latest/cephfs/) via [OpenStack Manila](https://wiki.openstack.org/wiki/Manila).
 - Slurm accounting using a MySQL database.
 - Monitoring integrated with Slurm jobs using Prometheus, ElasticSearch and Grafana.
-- A web-based portal from [OpenOndemand](https://openondemand.org/).
+- A web-based portal from [Open OnDemand](https://openondemand.org/).
 - Production-ready default Slurm configurations for access and memory limits.
 - [Packer](https://developer.hashicorp.com/packer)-based image build configurations for node images.
 
@@ -25,17 +25,18 @@ The default configuration in this repository may be used to create a cluster to
 - Persistent state backed by an OpenStack volume.
 - NFS-based shared file system backed by another OpenStack volume.
 
-Note that the OpenOndemand portal and its remote apps are not usable with this default configuration.
+Note that the Open OnDemand portal and its remote apps are not usable with this default configuration.
 
 It requires an OpenStack cloud, and an Ansible "deploy host" with access to that cloud.
 
 Before starting ensure that:
 - You have root access on the deploy host.
 - You can create instances using a Rocky 9 GenericCloud image (or an image based on that).
-    - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI. However the appliance will install the necessary packages if a GenericCloud image is used.
-- You have a SSH keypair defined in OpenStack, with the private key available on the deploy host.
+    - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI.
+- You have an SSH keypair defined in OpenStack, with the private key available on the deploy host.
 - Created instances have access to internet (note proxies can be setup through the appliance if necessary).
 - Created instances have accurate/synchronised time (for VM instances this is usually provided by the hypervisor; if not or for bare metal instances it may be necessary to configure a time service via the appliance).
+- Three security groups are present: ``default`` allowing intra-cluster communication, ``SSH`` allowing external access via SSH and ``HTTPS`` allowing access for Open OnDemand.
 
 ### Setup deploy host
 
@@ -55,21 +56,29 @@ You will also need to install [OpenTofu](https://opentofu.org/docs/intro/install
 
 ### Create a new environment
 
-Use the `cookiecutter` template to create a new environment to hold your configuration. In the repository root run:
+Run the following from the repository root to activate the venv:
 
     . venv/bin/activate
+
+Use the `cookiecutter` template to create a new environment to hold your configuration:
+
     cd environments
     cookiecutter skeleton
 
 and follow the prompts to complete the environment name and description.
 
-**NB:** In subsequent sections this new environment is refered to as `$ENV`.
+**NB:** In subsequent sections this new environment is referred to as `$ENV`.
+
+Go back to the root folder and activate the new environment:
 
-Now generate secrets for this environment:
+    cd ..
+    . environments/$ENV/activate
+
+And generate secrets for it:
 
     ansible-playbook ansible/adhoc/generate-passwords.yml
 
-### Define infrastructure configuration
+### Define and deploy infrastructure
 
 Create an OpenTofu variables file to define the required infrastructure, e.g.:
 
@@ -91,25 +100,34 @@ Create an OpenTofu variables file to define the required infrastructure, e.g.:
         }
     }
 
-Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables
-and descriptions see `environments/$ENV/terraform/terraform.tfvars`.
+Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables and descriptions see `environments/$ENV/terraform/terraform.tfvars`.
+
+To deploy this infrastructure, ensure the venv and the environment are [activated](#create-a-new-environment) and run:
 
-### Deploy appliance
+    export OS_CLOUD=openstack
+    cd environments/$ENV/terraform/
+    tofu init
+    tofu apply
+
+and follow the prompts. Note the OS_CLOUD environment variable assumes that OpenStack credentials are defined using a [clouds.yaml](https://docs.openstack.org/python-openstackclient/latest/configuration/index.html#clouds-yaml) file in a default location with the default cloud name of `openstack`.
+
+### Configure appliance
+
+To configure the appliance, ensure the venv and the environment are [activated](#create-a-new-environment) and run:
 
     ansible-playbook ansible/site.yml
 
-You can now log in to the cluster using:
+Once it completes you can log in to the cluster using:
 
     ssh rocky@$login_ip
 
 where the IP of the login node is given in `environments/$ENV/inventory/hosts.yml`
 
-
 ## Overview of directory structure
 
 - `environments/`: See [docs/environments.md](docs/environments.md).
-- `ansible/`: Contains the ansible playbooks to configure the infrastruture.
-- `packer/`: Contains automation to use Packer to build machine images for an enviromment - see the README in this directory for further information.
+- `ansible/`: Contains the ansible playbooks to configure the infrastructure.
+- `packer/`: Contains automation to use Packer to build machine images for an environment - see the README in this directory for further information.
 - `dev/`: Contains development tools.
 
 For further information see the [docs](docs/) directory.
diff --git a/ansible/.gitignore b/ansible/.gitignore
index 2ceeb596b..1cabb8ad8 100644
--- a/ansible/.gitignore
+++ b/ansible/.gitignore
@@ -58,4 +58,21 @@ roles/*
 !roles/squid/**
 !roles/tuned/
 !roles/tuned/**
-
+!roles/sssd/
+!roles/sssd/**
+!roles/sshd/
+!roles/sshd/**
+!roles/compute_init/
+!roles/compute_init/**
+!roles/k3s/
+!roles/k3s/**
+!roles/k9s/
+!roles/k9s/**
+!roles/lustre/
+!roles/lustre/**
+!roles/dnf_repos/
+!roles/dnf_repos/**
+!roles/pulp_site/
+!roles/pulp_site/**
+!roles/doca/
+!roles/doca/**
diff --git a/ansible/adhoc/deploy-pulp.yml b/ansible/adhoc/deploy-pulp.yml
new file mode 100644
index 000000000..2858d032b
--- /dev/null
+++ b/ansible/adhoc/deploy-pulp.yml
@@ -0,0 +1,26 @@
+# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
+
+- name: Add temporary pulp server host
+  hosts: localhost
+  tasks:
+  - ansible.builtin.add_host:
+      name: "{{ pulp_server }}"
+      group: "_pulp_host"
+
+- name: Install pulp on server and add to config
+  become: yes
+  hosts: _pulp_host
+  tasks:
+  - name: Install pulp
+    ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: install.yml
+      public: true
+
+  - name: Print Pulp endpoint
+    become: no
+    debug:
+      msg: | 
+        Server configured, override 'appliances_pulp_url' with
+          appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
+        in your environments
diff --git a/ansible/adhoc/rebuild.yml b/ansible/adhoc/rebuild.yml
index c30737fd6..9e7a3a770 100644
--- a/ansible/adhoc/rebuild.yml
+++ b/ansible/adhoc/rebuild.yml
@@ -16,3 +16,6 @@
     - command: "openstack server rebuild {{ instance_id | default(inventory_hostname) }}{% if rebuild_image is defined %} --image {{ rebuild_image }}{% endif %}"
       delegate_to: localhost
     - wait_for_connection:
+        delay: 60
+        timeout: 600
+
diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml
new file mode 100644
index 000000000..b2cd9a8c4
--- /dev/null
+++ b/ansible/adhoc/sync-pulp.yml
@@ -0,0 +1,10 @@
+- hosts: localhost
+  tasks:
+    - ansible.builtin.include_role:
+        name: pulp_site
+        tasks_from: sync.yml
+      vars:
+        pulp_site_target_arch: "x86_64"
+        pulp_site_target_distribution: "rocky"
+        pulp_site_target_distribution_version: "9.5"
+        pulp_site_target_distribution_version_major: "9"
diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml
index 18d159996..88d9274b3 100644
--- a/ansible/bootstrap.yml
+++ b/ansible/bootstrap.yml
@@ -110,6 +110,28 @@
         policy: "{{ selinux_policy }}"
       register: sestatus
 
+- hosts: sshd
+  tags: sshd
+  gather_facts: no
+  become: yes
+  tasks:
+    - name: Configure sshd
+      import_role:
+        name: sshd
+
+- hosts: dnf_repos
+  become: yes
+  tasks:
+  - name: Check that creds won't be leaked to users
+    ansible.builtin.assert:
+      that: dnf_repos_password is undefined
+      fail_msg: Passwords should not be templated into repofiles during configure, unset 'dnf_repos_password'
+    when: appliances_mode == 'configure'
+  - name: Replace system repos with pulp repos 
+    ansible.builtin.include_role:
+      name: dnf_repos
+      tasks_from: set_repos.yml
+
 # --- tasks after here require access to package repos ---
 - hosts: squid
   tags: squid
@@ -259,3 +281,11 @@
   tasks:
     - include_role:
         name: azimuth_cloud.image_utils.linux_ansible_init
+
+- hosts: k3s
+  become: yes
+  tags: k3s
+  tasks:
+    - ansible.builtin.include_role:
+        name: k3s
+        tasks_from: install.yml
diff --git a/ansible/ci/check_slurm.yml b/ansible/ci/check_slurm.yml
index d95c5bb5c..6507caf08 100644
--- a/ansible/ci/check_slurm.yml
+++ b/ansible/ci/check_slurm.yml
@@ -6,9 +6,9 @@
       shell: 'sinfo --noheader --format="%N %P %a %l %D %t" | sort' # using --format ensures we control whitespace: Partition,partition_state,max_jobtime,num_nodes,node_state,node_name
       register: sinfo
       changed_when: false
-      until: "'boot' not in sinfo.stdout_lines"
-      retries: 5
-      delay: 10
+      until: not ("boot" in sinfo.stdout or "idle*" in sinfo.stdout)
+      retries: 10
+      delay: 5
     - name: Check nodes have expected slurm state
       assert:
         that: sinfo.stdout_lines == expected_sinfo
diff --git a/ansible/cleanup.yml b/ansible/cleanup.yml
index 9c1373667..670a99b29 100644
--- a/ansible/cleanup.yml
+++ b/ansible/cleanup.yml
@@ -38,7 +38,7 @@
 
 - name: Cleanup /tmp
   command : rm -rf /tmp/*
-
+    
 - name: Get package facts
   package_facts:
 
@@ -61,5 +61,9 @@
       os: "{{ ansible_distribution }} {{ ansible_distribution_version }}"
       kernel: "{{ ansible_kernel }}"
       ofed: "{{ ansible_facts.packages['mlnx-ofa_kernel'].0.version | default('-') }}"
+      doca: "{{ ansible_facts.packages[doca_profile | default('doca-ofed') ].0.version | default('-') }}"
       cuda:  "{{ ansible_facts.packages['cuda'].0.version | default('-') }}"
       slurm-ohpc: "{{ ansible_facts.packages['slurm-ohpc'].0.version | default('-') }}"
+
+- name: Show image summary
+  command: cat /var/lib/image/image.json
diff --git a/ansible/disable-repos.yml b/ansible/disable-repos.yml
new file mode 100644
index 000000000..3e8022965
--- /dev/null
+++ b/ansible/disable-repos.yml
@@ -0,0 +1,7 @@
+- hosts: dnf_repos
+  become: yes
+  tasks:
+    - name: Disable pulp repos
+      ansible.builtin.include_role:
+        name: dnf_repos
+        tasks_from: disable_repos.yml
diff --git a/ansible/extras.yml b/ansible/extras.yml
index c32f51c32..72c76b3b1 100644
--- a/ansible/extras.yml
+++ b/ansible/extras.yml
@@ -24,8 +24,9 @@
   gather_facts: yes
   tags: cuda
   tasks:
-    - import_role:
+    - include_role:
         name: cuda
+        tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'install.yml' }}"
 
 - name: Persist hostkeys across rebuilds
   # Must be after filesystems.yml (for storage)
@@ -36,3 +37,31 @@
   tasks:
     - import_role:
         name: persist_hostkeys
+
+
+- name: Setup NFS export for compute node configuration
+  hosts: compute_init:!builder
+  # NB: has to be after eeesi and os-manila-mount
+  tags: compute_init
+  become: yes
+  tasks:
+    - include_role:
+        name: compute_init
+        tasks_from: export.yml
+
+- name: Install k9s
+  become: yes
+  hosts: k9s
+  tags: k9s
+  tasks:
+  - import_role:
+      name: k9s
+
+- hosts: extra_packages
+  become: yes
+  tags:
+   - extra_packages
+  tasks:
+  - name: Install additional packages
+    dnf:
+      name: "{{ appliances_extra_packages }}"
diff --git a/ansible/fatimage.yml b/ansible/fatimage.yml
index 4b5d59abe..5e9a47ca0 100644
--- a/ansible/fatimage.yml
+++ b/ansible/fatimage.yml
@@ -6,6 +6,9 @@
   tasks:
     - name: Report hostname (= final image name)
       command: hostname
+    - name: Report inventory groups
+      debug:
+        var: group_names
 
 - name: Run pre.yml hook
   vars:
@@ -14,8 +17,26 @@
   import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
   when: hook_path | exists
 
+- name: Sync pulp repos with upstream
+  hosts: pulp
+  tasks:
+  - ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: sync.yml
+      apply:
+        delegate_to: localhost
+    when: appliances_mode != 'configure'
+
 - import_playbook: bootstrap.yml
 
+- hosts: doca
+  become: yes
+  gather_facts: yes
+  tasks:
+    - name: Install NVIDIA DOCA
+      import_role:
+        name: doca
+
 - name: Run post-bootstrap.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
@@ -25,7 +46,7 @@
 
 - hosts: builder
   become: yes
-  gather_facts: no
+  gather_facts: yes
   tasks:
     # - import_playbook: iam.yml
     - name: Install FreeIPA client
@@ -33,6 +54,11 @@
         name: freeipa
         tasks_from: client-install.yml
       when: "'freeipa_client' in group_names"
+    - name: Install sssd
+      import_role:
+        name: sssd
+        tasks_from: install.yml
+      when: "'sssd' in group_names"
 
     # - import_playbook: filesystems.yml:
     - name: Install nfs packages
@@ -44,9 +70,24 @@
         name: stackhpc.os-manila-mount
         tasks_from: install.yml
       when: "'manila' in group_names"
+    - name: Install Lustre packages
+      include_role:
+        name: lustre
+        tasks_from: install.yml
+      when: "'lustre' in group_names"
 
 - import_playbook: extras.yml
 
+# TODO: is this the right place?
+- name: Install compute_init script
+  hosts: compute_init
+  tags: compute_init # tagged to allow running on cluster instances for dev
+  become: yes
+  tasks:
+    - include_role:
+        name: compute_init
+        tasks_from: install.yml
+
 - hosts: builder
   become: yes
   gather_facts: yes
@@ -57,6 +98,7 @@
         name: mysql
         tasks_from: install.yml
       when: "'mysql' in group_names"
+
     - name: OpenHPC
       include_role:
         name: stackhpc.openhpc
@@ -83,18 +125,25 @@
       import_role:
         name: openondemand
         tasks_from: vnc_compute.yml
+
       when: "'openondemand_desktop' in group_names"
+
     - name: Open Ondemand jupyter node
       import_role:
         name: openondemand
         tasks_from: jupyter_compute.yml
-      when: "'openondemand' in group_names"
+      when: "'openondemand_jupyter' in group_names"
+
+    - name: Install Apache PAM module # Extracted from start of roles/openondemand/tasks/pam_auth.yml to ensure only installed during build
+      yum:
+        name: mod_authnz_pam
 
     # - import_playbook: monitoring.yml:
     - import_role:
         name: opensearch
         tasks_from: install.yml
       when: "'opensearch' in group_names"
+
     # slurm_stats - nothing to do
     - import_role:
         name: filebeat
@@ -193,6 +242,8 @@
   import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
   when: hook_path | exists
 
+- import_playbook: disable-repos.yml
+
 - hosts: builder
   become: yes
   gather_facts: yes
diff --git a/ansible/filesystems.yml b/ansible/filesystems.yml
index e1a782bad..4665c0f8f 100644
--- a/ansible/filesystems.yml
+++ b/ansible/filesystems.yml
@@ -24,3 +24,13 @@
   tasks:
     - include_role:
         name: stackhpc.os-manila-mount
+
+- name: Setup Lustre clients
+  hosts: lustre
+  become: true
+  tags: lustre
+  tasks:
+    - include_role:
+        name: lustre
+        # NB install is ONLY run in builder
+        tasks_from: configure.yml
diff --git a/ansible/filter_plugins/utils.py b/ansible/filter_plugins/utils.py
index f69d6f3f7..508f794cc 100644
--- a/ansible/filter_plugins/utils.py
+++ b/ansible/filter_plugins/utils.py
@@ -41,6 +41,9 @@ def to_ood_regex(items):
         eg {{ [compute-001, compute-002, control] | to_regex }} -> '(compute-\d+)|(control)'
     """
     
+    # NB: for python3.12+ the \d in this function & docstring
+    # need to be raw strings. See https://docs.python.org/3/reference/lexical_analysis.html
+
     # There's a python bug which means re.sub() can't use '\d' in the replacement so
     # have to do replacement in two stages:
     r = [re.sub(r"\d+", 'XBACKSLASHX', v) for v in items]
@@ -48,6 +51,11 @@ def to_ood_regex(items):
     r = ['(%s)' % v for v in r]
     return '|'.join(r)
 
+def appliances_repo_to_subpath(repo_entry):
+    """ Take an element from appliances_pulp_repos and convert it to a pulp path. This assumes that the remote and local pulp structures are the same
+    """
+    return repo_entry['path'] + '/' + repo_entry['timestamp']
+
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
@@ -63,4 +71,5 @@ def filters(self):
             'exists': exists,
             'warn': self.warn,
             'to_ood_regex': to_ood_regex,
+            'appliances_repo_to_subpath': appliances_repo_to_subpath
         }
diff --git a/ansible/iam.yml b/ansible/iam.yml
index 0286b9df3..857b8f840 100644
--- a/ansible/iam.yml
+++ b/ansible/iam.yml
@@ -40,3 +40,12 @@
       import_role:
         name: freeipa
         tasks_from: users.yml
+
+- hosts: sssd
+  become: yes
+  gather_facts: no
+  tags: sssd
+  tasks:
+    - name: Configure sssd
+      import_role:
+        name: sssd
diff --git a/ansible/roles/basic_users/README.md b/ansible/roles/basic_users/README.md
index 4b75100ca..65fdd2c4c 100644
--- a/ansible/roles/basic_users/README.md
+++ b/ansible/roles/basic_users/README.md
@@ -24,6 +24,7 @@ Role Variables
   - An additional key `sudo` may optionally be specified giving a string (possibly multiline) defining sudo rules to be templated.
   - Any other keys may present for other purposes (i.e. not used by this role).
 - `basic_users_groups`: Optional, default empty list. A list of mappings defining information for each group. Mapping keys/values are passed through as parameters to [ansible.builtin.group](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/group_module.html) and default values are as given there.
+- `basic_users_override_sssd`: Optional bool, default false. Whether to disable `sssd` when ensuring users/groups exist with this role. Permits creating local users/groups even if they clash with users provided via sssd (e.g. from LDAP). Ignored if host is not in group `sssd` as well. Note with this option active `sssd` will be stopped and restarted each time this role is run.
 
 Dependencies
 ------------
diff --git a/ansible/roles/basic_users/defaults/main.yml b/ansible/roles/basic_users/defaults/main.yml
index 9f34bdf4c..e6c6eafaa 100644
--- a/ansible/roles/basic_users/defaults/main.yml
+++ b/ansible/roles/basic_users/defaults/main.yml
@@ -7,3 +7,4 @@ basic_users_userdefaults:
   shell: "{{'/sbin/nologin' if 'control' in group_names else omit }}"
 basic_users_users: []
 basic_users_groups: []
+basic_users_override_sssd: false
diff --git a/ansible/roles/basic_users/tasks/main.yml b/ansible/roles/basic_users/tasks/main.yml
index c27d024b4..c6733fb89 100644
--- a/ansible/roles/basic_users/tasks/main.yml
+++ b/ansible/roles/basic_users/tasks/main.yml
@@ -7,7 +7,16 @@
     label: "{{ item.name }}"
   when:
     - "item.state | default('present') == 'absent'"
-  
+
+- name: Stop sssd if required
+  systemd:
+    name: sssd
+    state: stopped
+  register: _stop_sssd
+  when:
+    - "'sssd' in group_names"
+    - basic_users_override_sssd | bool
+
 - name: Create groups
   ansible.builtin.group: "{{ item }}"
   loop:  "{{ basic_users_groups }}"
@@ -19,6 +28,12 @@
     label: "{{ item.name }} [{{ item.state | default('present') }}]"
   register: basic_users_info
 
+- name: Restart sssd if required
+  systemd:
+    name: sssd
+    state: started
+  when: _stop_sssd is changed
+
 - name: Write supplied public key as authorized for SSH access
   authorized_key:
     user: "{{ item.name }}"
diff --git a/ansible/roles/cluster_infra/templates/resources.tf.j2 b/ansible/roles/cluster_infra/templates/resources.tf.j2
index 344137b62..69d001105 100644
--- a/ansible/roles/cluster_infra/templates/resources.tf.j2
+++ b/ansible/roles/cluster_infra/templates/resources.tf.j2
@@ -7,6 +7,19 @@ data "openstack_identity_auth_scope_v3" "scope" {
   name = "{{ cluster_name }}"
 }
 
+####
+#### Data resources
+####
+
+resource "terraform_data" "k3s_token" {
+    input = "{{ k3s_token }}"
+    lifecycle {
+        ignore_changes = [
+            input, # makes it a write-once value (set via Ansible)
+        ]
+    }   
+}
+
 #####
 ##### Security groups for the cluster
 #####
@@ -386,6 +399,8 @@ resource "openstack_compute_instance_v2" "login" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    control_address = openstack_compute_instance_v2.control.network[0].fixed_ip_v4
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
@@ -400,6 +415,7 @@ resource "openstack_compute_instance_v2" "control" {
 
   network {
     port = openstack_networking_port_v2.control.id
+    access_network = true
   }
 
   {% if cluster_storage_network is defined %}
@@ -455,7 +471,7 @@ resource "openstack_compute_instance_v2" "control" {
     {%- endif %}
     bootcmd:
     %{for volume in [openstack_blockstorage_volume_v3.state, {% if not cluster_home_manila_share | bool %} openstack_blockstorage_volume_v3.home {% endif %}]}
-    - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
+    - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${replace(substr(volume.id, 0, 20), "-", "*")}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
     %{endfor}
     mounts:
         - [LABEL=state, {{ appliances_state_dir }}, auto]
@@ -479,6 +495,7 @@ resource "openstack_compute_instance_v2" "control" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
@@ -548,6 +565,8 @@ resource "openstack_compute_instance_v2" "{{ partition.name }}" {
         ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
       {% endif %}
     {% endfor %} 
+    control_address = openstack_compute_instance_v2.control.network[0].fixed_ip_v4
+    k3s_token = "{{ k3s_token }}"
   }
 }
 
diff --git a/ansible/roles/compute_init/README.md b/ansible/roles/compute_init/README.md
new file mode 100644
index 000000000..db18034aa
--- /dev/null
+++ b/ansible/roles/compute_init/README.md
@@ -0,0 +1,133 @@
+# EXPERIMENTAL: compute-init
+
+Experimental / in-progress functionality to allow compute nodes to rejoin the
+cluster after a reboot.
+
+To enable this add compute nodes (or a subset of them into) the `compute_init`
+group.
+
+This works as follows:
+1. During image build, an ansible-init playbook and supporting files
+(e.g. templates, filters, etc) are installed.
+2. Cluster instances are created as usual; the above compute-init playbook does
+not run.
+3. The `site.yml` playbook is run as usual to configure all the instances into
+a cluster. In addition, with `compute-init` enabled, a `/exports/cluster` NFS
+share is created on the control node containing:
+    - an /etc/hosts file for the cluster
+    - Hostvars for each compute node
+4. On reboot of a compute node, ansible-init runs the compute-init playbook
+which:
+    a. Checks whether the `enable_compute` metadata flag is set, and exits if
+       not.
+    b. Tries to mount the above `/exports/cluster` NFS share from the control
+       node, and exits if it cannot.
+    c. Configures itself using the exported hostvars, depending on the
+       `enable_*` flags set in metadata.
+    d. Issues an `scontrol` command to resume the node (because Slurm will
+       consider it as "unexpectedly rebooted").
+
+The check in 4b. above is what prevents the compute-init script from trying
+to configure the node before the services on the control node are available
+(which requires running the site.yml playbook).
+
+The following roles/groups are currently fully functional:
+- `resolv_conf`: all functionality
+- `etc_hosts`: all functionality
+- `nfs`: client functionality only
+- `manila`: all functionality
+- `basic_users`: all functionality, assumes home directory already exists on
+  shared storage
+- `eessi`: all functionality, assumes `cvmfs_config` is the same on control
+  node and all compute nodes.
+- `openhpc`: all functionality
+
+The above may be enabled by setting the compute_init_enable property on the
+terraform compute variable.
+
+# Development/debugging
+
+To develop/debug changes to the compute script without actually having to build
+a new image:
+
+1. Deploy a cluster using tofu and ansible/site.yml as normal. This will
+   additionally configure the control node to export compute hostvars over NFS.
+   Check the cluster is up.
+
+2. Reimage the compute nodes:
+
+        ansible-playbook --limit compute ansible/adhoc/rebuild.yml
+
+3. Add metadata to a compute node e.g. via Horizon to turn on compute-init
+   playbook functionality.
+
+4. Fake an image build to deploy the compute-init playbook:
+
+        ansible-playbook ansible/fatimage.yml --tags compute_init
+
+    NB: This will also re-export the compute hostvars, as the nodes are not
+    in the builder group, which conveniently means any changes made to that
+    play also get picked up.
+
+5. Fake a reimage of compute to run ansible-init and the compute-init playbook:
+
+    On compute node where metadata was added:
+
+        [root@rl9-compute-0 rocky]# rm -f /var/lib/ansible-init.done && systemctl restart ansible-init
+        [root@rl9-compute-0 rocky]# systemctl status ansible-init
+
+    Use `systemctl status ansible-init` to view stdout/stderr from Ansible.
+
+Steps 4/5 can be repeated with changes to the compute script. If required,
+reimage the compute node(s) first as in step 2 and/or add additional metadata
+as in step 3.
+
+
+# Design notes
+- Duplicating code in roles into the `compute-init` script is unfortunate, but
+  does allow developing this functionality without wider changes to the
+  appliance.
+
+- In general, we don't want to rely on NFS export. So should e.g. copy files
+  from this mount ASAP in the compute-init script. TODO:
+
+- There are a couple of approaches to supporting existing roles using `compute-init`:
+
+  1. Control node copies files resulting from role into cluster exports,
+     compute-init copies to local disk. Only works if files are not host-specific
+     Examples: etc_hosts, eessi config?
+  
+  2. Re-implement the role. Works if the role vars are not too complicated,
+     (else they all need to be duplicated in compute-init). Could also only
+     support certain subsets of role functionality or variables
+     Examples: resolv_conf, stackhpc.openhpc
+
+- Some variables are defined using hostvars from other nodes, which aren't
+  available v the current approach:
+
+    ```
+    [root@rl9-compute-0 rocky]# grep hostvars /mnt/cluster/hostvars/rl9-compute-0/hostvars.yml
+        "grafana_address": "{{ hostvars[groups['grafana'].0].api_address }}",
+        "grafana_api_address": "{{ hostvars[groups['grafana'].0].internal_address }}",
+        "mysql_host": "{{ hostvars[groups['mysql'] | first].api_address }}",
+        "nfs_server_default": "{{ hostvars[groups['control'] | first ].internal_address }}",
+        "openhpc_slurm_control_host": "{{ hostvars[groups['control'].0].api_address }}",
+        "openondemand_address": "{{ hostvars[groups['openondemand'].0].api_address if groups['openondemand'] | count > 0 else '' }}",
+        "openondemand_node_proxy_directives": "{{ _opeonondemand_unset_auth if (openondemand_auth == 'basic_pam' and 'openondemand_host_regex' and groups['grafana'] | length > 0 and hostvars[ groups['grafana']  | first]._grafana_auth_is_anonymous) else '' }}",
+        "openondemand_servername": "{{ hostvars[ groups['openondemand'] | first].ansible_host }}",
+        "prometheus_address": "{{ hostvars[groups['prometheus'].0].api_address }}",
+            "{{ hostvars[groups['freeipa_server'].0].ansible_host }}"
+    ```
+
+    More generally, there is nothing to stop any group var depending on a
+    "{{ hostvars[] }}" interpolation ...
+
+    Only `nfs_server_default` and `openhpc_slurm_control_host` are of concern
+    for compute nodes - both of these indirect via `api_address` to
+    `inventory_hostname`. This has been worked around by replacing this with
+    "{{ groups['control'] | first }}" which does result in the control node
+    inventory hostname when templating.
+
+    Note that although `groups` is defined in the templated hostvars, when
+    the hostvars are loaded using `include_vars:` is is ignored as it is a
+    "magic variable" determined by ansible itself and cannot be set.
diff --git a/ansible/roles/compute_init/files/compute-init.yml b/ansible/roles/compute_init/files/compute-init.yml
new file mode 100644
index 000000000..430e2cf65
--- /dev/null
+++ b/ansible/roles/compute_init/files/compute-init.yml
@@ -0,0 +1,285 @@
+---
+
+- name: Compute node initialisation
+  hosts: localhost
+  become: yes
+  vars:
+    os_metadata: "{{ lookup('url', 'http://169.254.169.254/openstack/latest/meta_data.json') | from_json }}"
+    server_node_ip: "{{ os_metadata.meta.control_address }}"
+    enable_compute: "{{ os_metadata.meta.compute | default(false) | bool }}"
+    enable_resolv_conf: "{{ os_metadata.meta.resolv_conf | default(false) | bool }}"
+    enable_etc_hosts: "{{ os_metadata.meta.etc_hosts | default(false) | bool }}"
+    enable_nfs: "{{ os_metadata.meta.nfs | default(false) | bool }}"
+    enable_manila: "{{ os_metadata.meta.manila | default(false) | bool }}"
+    enable_basic_users: "{{ os_metadata.meta.basic_users | default(false) | bool }}"
+    enable_eessi: "{{ os_metadata.meta.eessi | default(false) | bool }}"
+
+    # TODO: "= role defaults" - could be moved to a vars_file: on play with similar precedence effects
+    resolv_conf_nameservers: []
+
+    nfs_client_mnt_point: "/mnt"
+    nfs_client_mnt_options:
+    nfs_client_mnt_state: mounted
+    nfs_configurations:
+    nfs_enable:
+      clients: false
+
+    # openhpc: no defaults required
+
+    os_manila_mount_shares: []
+    os_manila_mount_ceph_conf_path: /etc/ceph
+    os_manila_mount_state: mounted
+    os_manila_mount_opts:
+      - x-systemd.device-timeout=30
+      - x-systemd.mount-timeout=30
+      - noatime
+      - _netdev # prevents mount blocking early boot before networking available
+      - rw
+
+    basic_users_groups: []
+    basic_users_manage_homedir: false # homedir must already exist on shared filesystem
+    basic_users_userdefaults:
+      state: present
+      create_home: "{{ basic_users_manage_homedir }}"
+      generate_ssh_key:  "{{ basic_users_manage_homedir }}"
+      ssh_key_comment: "{{ item.name }}"
+    basic_users_users: []
+
+  tasks:
+    - block:
+        - name: Report skipping initialization if not compute node
+          # meta: end_play produces no output
+          debug:
+            msg: "Skipping compute initialization: Metadata enable_compute is not true"
+        
+        - meta: end_play
+      when: not enable_compute
+
+    - name: Ensure the mount directory exists
+      file:
+        path: /mnt/cluster
+        state: directory
+        owner: root
+        group: root
+        mode: u=rwX,go= # is sensitive
+    
+    - name: Mount /mnt/cluster
+      mount:
+        path: /mnt/cluster
+        src: "{{ server_node_ip }}:/exports/cluster"
+        fstype: nfs
+        opts: ro,sync
+        state: mounted
+      register: _mount_mnt_cluster
+      ignore_errors: true
+      # TODO: add some retries here?
+
+    - block:
+        - name: Report skipping initialization if cannot mount nfs
+          # meta: end_play produces no output
+          debug:
+            msg: "Skipping compute initialization: Failed to mount /exports/cluster from control node {{ server_node_ip }}"
+        
+        - meta: end_play
+      when: _mount_mnt_cluster.failed
+
+    - name: Load hostvars from NFS
+      # this is higher priority than vars block = normal ansible's hostvars
+      include_vars:
+        file: "/mnt/cluster/hostvars/{{ ansible_hostname }}/hostvars.yml" # can't use inventory_hostname
+
+      # TODO: should /mnt/cluster now be UNMOUNTED to avoid future hang-ups?
+
+    - name: Configure resolve.conf
+      block:
+        - name: Set nameservers in /etc/resolv.conf
+          ansible.builtin.template:
+            src: resolv.conf.j2
+            dest: /etc/resolv.conf
+            owner: root
+            group: root
+            mode: u=rw,og=r
+
+        - name: Disable NetworkManager control of resolv.conf
+          ansible.builtin.copy:
+            src: files/NetworkManager-dns-none.conf
+            dest: /etc/NetworkManager/conf.d/90-dns-none.conf
+            owner: root
+            group: root
+            mode: u=rw,og=r
+          register: _copy_nm_config
+
+        - name: Reload NetworkManager
+          ansible.builtin.systemd:
+            name: NetworkManager
+            state: reloaded
+          when: _copy_nm_config.changed | default(false)
+      when: enable_resolv_conf
+
+    - name: Copy cluster /etc/hosts
+      copy:
+        src: /mnt/cluster/hosts
+        dest: /etc/hosts
+        owner: root
+        group: root
+        mode: 0644
+      when: enable_etc_hosts
+
+    # NFS client mount
+    - name: If nfs-clients is present
+      include_tasks: tasks/nfs-clients.yml
+      when:
+        - enable_nfs
+        - nfs_enable.clients | bool or ('nfs_enable' in item and item.nfs_enable.clients | bool)
+      loop: "{{ nfs_configurations }}"
+
+    - name: Manila mounts
+      block:
+        - name: Read manila share info from nfs file
+          include_vars:
+            file: /mnt/cluster/manila_share_info.yml
+          no_log: true # contains secrets
+        
+        - name: Ensure Ceph configuration directory exists
+          ansible.builtin.file:
+            path: "{{ os_manila_mount_ceph_conf_path }}"
+            state: directory
+            mode: "0755"
+            owner: root
+            group: root
+
+        - name: Configure ceph.conf using os_manila_mount_host
+          ansible.builtin.template:
+            src: ceph.conf.j2
+            dest: "{{ os_manila_mount_ceph_conf_path }}/ceph.conf"
+            owner: root
+            group: root
+            mode: "0600"
+
+        - name: Ensure mount directory exists
+          ansible.builtin.file:
+            path: "{{ item.mount_path }}"
+            state: directory
+            owner: "{{ item.mount_user | default(omit) }}"
+            group: "{{ item.mount_group | default(omit) }}"
+            mode: "{{ item.mount_mode | default(omit) }}"
+          loop: "{{ os_manila_mount_shares }}"
+          loop_control:
+            label: "{{ item.share_name }}"
+
+        - name: Write Ceph client keyring
+          ansible.builtin.template:
+            src: ceph.keyring.j2
+            dest: "{{ os_manila_mount_ceph_conf_path }}/ceph.client.{{ item.share_user }}.keyring"
+            mode: "0600"
+            owner: root
+            group: root
+          loop: "{{ os_manila_mount_share_info }}"
+          loop_control:
+            label: "{{ item.share_name }}"
+
+        - name: Mount the Ceph share
+          ansible.posix.mount:
+            path: "{{ item[0].mount_path }}"
+            src: "{{ item[1].host }}:{{ item[1].export }}"
+            fstype: ceph
+            opts: "name={{ item[1].share_user }},{{ (item[0].mount_opts | default(os_manila_mount_opts)) | join(',') }}"
+            # NB share_user is looked up here in case of autodetection
+            state: "{{ item[0].mount_state | default(os_manila_mount_state) }}"
+          loop: "{{ os_manila_mount_shares | zip(os_manila_mount_share_info) }}"
+          loop_control:
+            label: "{{ item[0].share_name }}"
+
+        - name: Ensure mounted directory has correct permissions
+          ansible.builtin.file:
+            path: "{{ item.mount_path }}"
+            state: directory
+            owner: "{{ item.mount_user | default(omit) }}"
+            group: "{{ item.mount_group | default(omit) }}"
+            mode: "{{ item.mount_mode | default(omit) }}"
+          loop: "{{ os_manila_mount_shares }}"
+          loop_control:
+            label: "{{ item.share_name }}"
+          when: item.mount_state | default(os_manila_mount_state) in ['mounted' or 'ephemeral']
+      when:
+        - enable_manila
+        - os_manila_mount_shares | length > 0
+
+    - name: Basic users
+      block:
+        - name: Create groups
+          ansible.builtin.group: "{{ item }}"
+          loop:  "{{ basic_users_groups }}"
+
+        - name: Create users
+          user: "{{ basic_users_userdefaults | combine(item) | filter_user_params() }}"
+          loop: "{{ basic_users_users }}"
+          loop_control:
+            label: "{{ item.name }} [{{ item.state | default('present') }}]"
+          register: basic_users_info
+
+        - name: Write sudo rules
+          blockinfile:
+            path: /etc/sudoers.d/80-{{ item.name}}-user
+            block: "{{ item.sudo }}"
+            create: true
+          loop: "{{ basic_users_users }}"
+          loop_control:
+            label: "{{ item.name }}"
+          when: "'sudo' in item"
+      when: enable_basic_users
+
+    - name: EESSI
+      block:
+        - name: Copy cvmfs config
+          copy:
+            src: /mnt/cluster/cvmfs/default.local
+            dest: /etc/cvmfs/default.local
+            owner: root
+            group: root
+            mode: 0644
+
+        - name: Ensure CVMFS config is setup
+          command:
+            cmd: "cvmfs_config setup"
+      when: enable_eessi
+
+    # NB: don't need conditional block on enable_compute as have already exited
+    # if not the case
+    - name: Write Munge key
+      copy:
+        content: "{{ openhpc_munge_key }}"
+        dest: "/etc/munge/munge.key"
+        owner: munge
+        group: munge
+        mode: 0400
+
+    - name: Set slurmctld location for configless operation
+      lineinfile:
+        path: /etc/sysconfig/slurmd
+        line: "SLURMD_OPTIONS='--conf-server {{ openhpc_slurm_control_host_address | default(openhpc_slurm_control_host) }}'"
+        regexp: "^SLURMD_OPTIONS="
+        create: yes
+        owner: root
+        group: root
+        mode: 0644
+
+    - name: Ensure Munge service state
+      service:
+        name: munge
+        enabled: true
+        state: started
+
+    - name: Ensure slurmd service state
+      service:
+        name: slurmd
+        enabled: true
+        state: started
+
+    - name: Ensure node is resumed
+      # TODO: consider if this is always safe for all job states?
+      command: scontrol update state=resume nodename={{ ansible_hostname }}
+      register: _scontrol_update
+      failed_when:
+        - _scontrol_update.rc > 0
+        - "'slurm_update error: Invalid node state specified' not in _scontrol_update.stderr"
diff --git a/ansible/roles/compute_init/tasks/export.yml b/ansible/roles/compute_init/tasks/export.yml
new file mode 100644
index 000000000..12b648f6e
--- /dev/null
+++ b/ansible/roles/compute_init/tasks/export.yml
@@ -0,0 +1,67 @@
+- name: Ensure the /exports/cluster directory exists
+  file:
+    path: /exports/cluster
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwX,go=
+  run_once: true
+  delegate_to: "{{ groups['control'] | first }}"
+
+- name: Copy /etc/hosts to /exports/cluster
+  copy:
+    src: /etc/hosts
+    dest: /exports/cluster/hosts
+    owner: root
+    group: root
+    mode: u=rw,go=
+    remote_src: true
+  run_once: true
+  delegate_to: "{{ groups['control'] | first }}"
+
+- name: Create hostvars directory
+  file:
+    path: /exports/cluster/hostvars/{{ inventory_hostname }}/
+    state: directory
+    mode: u=rwX,go=
+    # TODO: owner,mode,etc
+  delegate_to: "{{ groups['control'] | first }}"
+
+- name: Template out hostvars
+  template:
+    src: hostvars.yml.j2
+    dest: /exports/cluster/hostvars/{{ inventory_hostname }}/hostvars.yml
+    mode: u=rw,go=
+  delegate_to: "{{ groups['control'] | first }}"
+
+- name: Copy manila share info to /exports/cluster
+  copy:
+    content: "{{ os_manila_mount_share_info_var | to_nice_yaml }}"
+    dest: /exports/cluster/manila_share_info.yml
+  run_once: true
+  delegate_to: "{{ groups['control'] | first }}"
+  when: os_manila_mount_share_info is defined
+  vars:
+    os_manila_mount_share_info_var:
+      os_manila_mount_share_info: "{{ os_manila_mount_share_info }}"
+
+- name: Ensure /exports/cluster/cvmfs directory exists
+  file:
+    path: /exports/cluster/cvmfs
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  run_once: true
+  delegate_to: "{{ groups['control'] | first }}"
+
+- name: Copy EESSI CVMFS config to /exports/cluster
+  copy:
+    src: /etc/cvmfs/default.local
+    dest: /exports/cluster/cvmfs/default.local
+    owner: root
+    group: root
+    mode: 0644
+    remote_src: true
+  run_once: true
+  delegate_to: "{{ groups['control'] | first }}"
diff --git a/ansible/roles/compute_init/tasks/install.yml b/ansible/roles/compute_init/tasks/install.yml
new file mode 100644
index 000000000..bbcbf133f
--- /dev/null
+++ b/ansible/roles/compute_init/tasks/install.yml
@@ -0,0 +1,53 @@
+---
+
+- name: Ensure directories exist
+  file:
+    path: "/etc/ansible-init/playbooks/{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  loop:
+    - templates
+    - files
+    - library
+    - filter_plugins
+    - tasks
+
+- name: Inject files from roles
+  copy:
+    src: '{{ item.src }}'
+    dest: '/etc/ansible-init/playbooks/{{ item.dest }}'
+    owner: root
+    group: root
+    mode: 0644
+  loop:
+    - src: ../../resolv_conf/templates/resolv.conf.j2
+      dest: templates/resolv.conf.j2
+    - src: ../../stackhpc.os-manila-mount/templates/ceph.conf.j2
+      dest: templates/ceph.conf.j2
+    - src: ../../stackhpc.os-manila-mount/templates/ceph.keyring.j2
+      dest: templates/ceph.keyring.j2
+    - src: ../../resolv_conf/files/NetworkManager-dns-none.conf
+      dest: files/NetworkManager-dns-none.conf
+    - src: ../../basic_users/filter_plugins/filter_keys.py
+      dest: filter_plugins/filter_keys.py
+    - src: ../../stackhpc.nfs/tasks/nfs-clients.yml
+      dest: tasks/nfs-clients.yml
+
+- name: Add filter_plugins to ansible.cfg
+  lineinfile:
+    path: /etc/ansible-init/ansible.cfg
+    line: "filter_plugins = /etc/ansible-init/filter_plugins"
+    state: present
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Add compute initialisation playbook
+  copy:
+    src: compute-init.yml
+    dest: /etc/ansible-init/playbooks/1-compute-init.yml
+    owner: root
+    group: root
+    mode: 0644
diff --git a/ansible/roles/compute_init/templates/hostvars.yml.j2 b/ansible/roles/compute_init/templates/hostvars.yml.j2
new file mode 100644
index 000000000..7d4351b44
--- /dev/null
+++ b/ansible/roles/compute_init/templates/hostvars.yml.j2
@@ -0,0 +1 @@
+{{ hostvars[inventory_hostname] | to_nice_json }}
\ No newline at end of file
diff --git a/ansible/roles/cuda/README.md b/ansible/roles/cuda/README.md
index 141e7b80d..be6439cd5 100644
--- a/ansible/roles/cuda/README.md
+++ b/ansible/roles/cuda/README.md
@@ -1,6 +1,6 @@
 # cuda
 
-Install NVIDIA CUDA. The CUDA binaries are added to the PATH for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
+Install NVIDIA drivers and optionally CUDA packages. CUDA binaries are added to the `$PATH` for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
 
 ## Prerequisites
 
@@ -8,8 +8,8 @@ Requires OFED to be installed to provide required kernel-* packages.
 
 ## Role Variables
 
-- `cuda_distro`: Optional. Default `rhel8`.
-- `cuda_repo`: Optional. Default `https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo`
-- `cuda_driver_stream`: Optional. The default value `default` will, on first use of this role, enable the dkms-flavour `nvidia-driver` DNF module stream with the current highest version number. The `latest-dkms` stream is not enabled, and subsequent runs of the role will *not* change the enabled stream, even if a later version has become available. Changing this value once an `nvidia-driver` stream has been enabled raises an error. If an upgrade of the `nvidia-driver` module is required, the currently-enabled stream and all packages should be manually removed.
+- `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
+- `cuda_nvidia_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
 - `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds']`.
+- `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.
diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index 33a25d9b4..05f1e093d 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -1,7 +1,6 @@
-cuda_distro: "rhel{{ ansible_distribution_major_version }}"
-cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
-cuda_driver_stream: default
-cuda_package_version: 'latest'
+cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
+cuda_nvidia_driver_stream: '560-open' # 565-open has problems with cuda packages
+cuda_package_version: '12.6.3-1'
 cuda_packages:
   - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
   - nvidia-gds
diff --git a/ansible/roles/cuda/tasks/main.yml b/ansible/roles/cuda/tasks/install.yml
similarity index 60%
rename from ansible/roles/cuda/tasks/main.yml
rename to ansible/roles/cuda/tasks/install.yml
index 22f8e9e8e..51c92a0d3 100644
--- a/ansible/roles/cuda/tasks/main.yml
+++ b/ansible/roles/cuda/tasks/install.yml
@@ -1,7 +1,7 @@
 
 # Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#redhat8-installation
 
-- name: Check for OFED
+- name: Check for OFED/DOCA
   command:
     cmd: dnf list --installed rdma-core
   register: _dnf_rdma_core
@@ -10,41 +10,53 @@
 - name: Assert OFED installed
   assert: 
     that: "'mlnx' in _dnf_rdma_core.stdout"
-    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED installed?"
+    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED/DOCA installed?"
 
 - name: Install cuda repo
   get_url:
-    dest: "/etc/yum.repos.d/cuda-{{ cuda_distro }}.repo"
-    url: "{{ cuda_repo }}"
+    dest: "/etc/yum.repos.d/cuda-rhel{{ ansible_distribution_major_version }}.repo"
+    url: "{{ cuda_repo_url }}"
 
 - name: Check if nvidia driver module is enabled
-  shell:
-    cmd: dnf module list --enabled nvidia-driver
+  ansible.builtin.command: dnf module list --enabled nvidia-driver
   changed_when: false
   failed_when: false
   register: _cuda_driver_module_enabled
 
 - name: Enable nvidia driver module
-  ansible.builtin.command: "dnf module enable -y nvidia-driver:open-dkms"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ cuda_nvidia_driver_stream }}"
   register: _cuda_driver_module_enable
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
+- name: Check if nvidia driver module is installed
+  ansible.builtin.command: dnf module list --installed nvidia-driver
+  changed_when: false
+  failed_when: false
+  register: _cuda_driver_module_installed
+
 - name: Install nvidia drivers
   ansible.builtin.command: dnf module install -y nvidia-driver
   register: _cuda_driver_install
-  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+  when: "'No matching Modules to list' in _cuda_driver_module_installed.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
+- name: Check kernel has not been modified
+  assert:
+    that: "'kernel ' not in _cuda_driver_install.stdout | default('')" # space ensures we don't flag e.g. kernel-devel-matched
+    fail_msg: "{{ _cuda_driver_install.stdout_lines | default([]) | select('search', 'kernel ') }}"
+
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"
+  when: cuda_package_version != 'none'
   register: cuda_package_install
 
 - name: Add cuda binaries to path
   lineinfile:
     path: /etc/profile.d/sh.local
     line: 'export PATH=$PATH:$(ls -1d /usr/local/cuda-* | sort -V | tail -1)/bin'
+  when: cuda_package_version != 'none'
 
 - name: Enable NVIDIA Persistence Daemon
   systemd:
@@ -60,3 +72,4 @@
 - name: Wait for hosts to be reachable
   wait_for_connection:
     sleep: 15
+  when: cuda_package_install.changed
diff --git a/ansible/roles/cuda/tasks/runtime.yml b/ansible/roles/cuda/tasks/runtime.yml
new file mode 100644
index 000000000..c16a48c6f
--- /dev/null
+++ b/ansible/roles/cuda/tasks/runtime.yml
@@ -0,0 +1,5 @@
+- name: Ensure NVIDIA Persistence Daemon state
+  systemd:
+    name: nvidia-persistenced
+    enabled: true
+    state: "{{ cuda_persistenced_state }}"
diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml
new file mode 100644
index 000000000..6d41046ec
--- /dev/null
+++ b/ansible/roles/dnf_repos/defaults/main.yml
@@ -0,0 +1,48 @@
+dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
+dnf_repos_username: "{{ omit }}"
+dnf_repos_password: "{{ omit }}"
+
+dnf_repos_filenames:
+  '8':
+    baseos: 'Rocky-BaseOS'
+    appstream: 'Rocky-AppStream'
+    crb: 'Rocky-PowerTools'
+    extras: 'Rocky-Extras'
+  '9':
+    baseos: 'rocky'
+    appstream: 'rocky'
+    crb: 'rocky'
+    extras: 'rocky-extras'
+
+dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
+
+# epel installed separately
+dnf_repos_default_repolist:
+- file: "{{ dnf_repos_version_filenames.baseos }}"
+  name: baseos
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.baseos[ansible_distribution_version] | appliances_repo_to_subpath }}"
+- file: "{{ dnf_repos_version_filenames.appstream }}"
+  name: appstream
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.appstream[ansible_distribution_version] | appliances_repo_to_subpath }}"
+- file: "{{ dnf_repos_version_filenames.crb }}"
+  name: "{{ 'powertools' if ansible_distribution_major_version == '8' else 'crb' }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.crb[ansible_distribution_version] | appliances_repo_to_subpath }}"
+- file: "{{ dnf_repos_version_filenames.extras }}"
+  name: extras
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.extras[ansible_distribution_version] | appliances_repo_to_subpath }}"
+- file: ceph
+  name: Ceph
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+
+dnf_repos_openhpc_repolist:
+- name: OpenHPC
+  file: OpenHPC
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_base[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+- name: OpenHPC-updates
+  file: OpenHPC
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+
+dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) }}"
+
+dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+dnf_repos_epel_description: "epel"
diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml
new file mode 100644
index 000000000..2dbacc262
--- /dev/null
+++ b/ansible/roles/dnf_repos/tasks/disable_repos.yml
@@ -0,0 +1,18 @@
+---
+- name: Disable Pulp repos
+  ansible.builtin.yum_repository:
+    file: "{{ item.file }}"
+    name: "{{ item.name }}"
+    baseurl: "{{ item.base_url }}"
+    description: "{{ item.name }}"
+    enabled: false
+  loop: "{{ dnf_repos_repolist }}"
+
+- name: Disable EPEL repo
+  ansible.builtin.yum_repository:
+    name: epel
+    file: epel
+    description: "{{ dnf_repos_epel_description }}"
+    baseurl: "{{ dnf_repos_epel_baseurl }}"
+    gpgcheck: false
+    enabled: false
diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml
new file mode 100644
index 000000000..c9fcb0c07
--- /dev/null
+++ b/ansible/roles/dnf_repos/tasks/set_repos.yml
@@ -0,0 +1,27 @@
+---
+
+- name: Replace system repos with Pulp repos
+  ansible.builtin.yum_repository:
+    file: "{{ item.file }}"
+    name: "{{ item.name }}"
+    baseurl: "{{ item.base_url }}"
+    description: "{{ item.name }}"
+    username: "{{ dnf_repos_username }}"
+    password: "{{ dnf_repos_password }}"
+    gpgcheck: false
+  loop: "{{ dnf_repos_repolist }}"
+
+- name: Install epel-release
+  # done so that roles installing epel via epel-release don't over-write our changes to the epel repo
+  ansible.builtin.dnf:
+    name: epel-release
+
+- name: Use Pulp EPEL repo
+  ansible.builtin.yum_repository:
+    name: epel
+    file: epel
+    description: "{{ dnf_repos_epel_description }}"
+    gpgcheck: false
+    baseurl: "{{ dnf_repos_epel_baseurl }}"
+    username: "{{ dnf_repos_username }}"
+    password: "{{ dnf_repos_password }}"
diff --git a/ansible/roles/doca/README.md b/ansible/roles/doca/README.md
new file mode 100644
index 000000000..5f898add5
--- /dev/null
+++ b/ansible/roles/doca/README.md
@@ -0,0 +1,12 @@
+# doca
+
+Install [NVIDIA DOCA](https://docs.nvidia.com/doca/sdk/index.html).
+
+This role is not idempotent and is only intended to be run during an image build. It builds DOCA kernel modules to match the installed kernel and then installs these
+plus the selected DOCA packages.
+
+## Role Variables
+
+- `doca_version`: Optional. String giving doca version.
+- `doca_profile`: Optional. Name of [profile](https://docs.nvidia.com/doca/sdk/nvidia+doca+profiles/index.html) defining subset of DOCA to install. Default is `doca-ofed`.
+- `doca_repo_url`: Optional. URL of DOCA repository. Default is appropriate upstream public repository for DOCA version, distro version and architecture.
diff --git a/ansible/roles/doca/defaults/main.yml b/ansible/roles/doca/defaults/main.yml
new file mode 100644
index 000000000..66437cd04
--- /dev/null
+++ b/ansible/roles/doca/defaults/main.yml
@@ -0,0 +1,3 @@
+doca_version: '2.9.1' # 2.9 is LTS, last to support ConnectX-4, 3 years for bug fixes and CVE updates
+doca_profile: doca-ofed
+doca_repo_url: "https://linux.mellanox.com/public/repo/doca/{{ doca_version }}/rhel{{ ansible_distribution_version }}/{{ ansible_architecture }}/"
diff --git a/ansible/roles/doca/tasks/install-kernel-devel.yml b/ansible/roles/doca/tasks/install-kernel-devel.yml
new file mode 100644
index 000000000..6a1943a32
--- /dev/null
+++ b/ansible/roles/doca/tasks/install-kernel-devel.yml
@@ -0,0 +1,24 @@
+- name: Get installed kernels
+  command: dnf list --installed kernel
+  register: _ofed_dnf_kernels
+  changed_when: false
+
+- name: Determine running kernel
+  command: uname -r # e.g. 4.18.0-513.18.1.el8_9.x86_64
+  register: _ofed_loaded_kernel
+  changed_when: false
+
+- name: Check current kernel is newest installed
+  assert:
+    that: _ofed_kernel_current == _ofed_dnf_kernels_newest
+    fail_msg: "Kernel {{ _ofed_loaded_kernel.stdout }} is loaded but newer {{ _ofed_dnf_kernels_newest }} is installed: consider rebooting?"
+  vars:
+    _ofed_kernel_current: >-
+      {{ _ofed_loaded_kernel.stdout | regex_replace('\.(?:.(?!\.))+$', '') | regex_replace('\.(?:.(?!\.))+$', '') }}
+    _ofed_dnf_kernels_newest: >-
+      {{ _ofed_dnf_kernels.stdout_lines[1:] | map('split') | map(attribute=1) | map('regex_replace', '\.(?:.(?!\.))+$', '') | community.general.version_sort | last }}
+    # dnf line format e.g. "kernel.x86_64  4.18.0-513.18.1.el8_9   @baseos  "
+
+- name: Install matching kernel-devel package
+  dnf:
+    name: "kernel-devel-{{ _ofed_loaded_kernel.stdout | trim }}"
diff --git a/ansible/roles/doca/tasks/install.yml b/ansible/roles/doca/tasks/install.yml
new file mode 100644
index 000000000..d26fda79e
--- /dev/null
+++ b/ansible/roles/doca/tasks/install.yml
@@ -0,0 +1,51 @@
+- import_tasks: install-kernel-devel.yml
+
+- name: Install DOCA repo
+  ansible.builtin.yum_repository:
+    name: doca
+    file: doca
+    description: DOCA Online Repo
+    baseurl: "{{ doca_repo_url }}"
+    enabled: true
+    gpgcheck: false
+
+- name: Install doca-extra package
+  ansible.builtin.dnf:
+    name: doca-extra
+
+- name: Build DOCA kernel modules
+  ansible.builtin.shell:
+    cmd: /opt/mellanox/doca/tools/doca-kernel-support
+  register: _doca_kernel_build
+
+
+- name: Find generated doca-kernel-repo
+  ansible.builtin.shell: 'find /tmp/DOCA.* -name doca-kernel-repo-*'
+  register: _doca_kernel_repo # e.g. /tmp/DOCA.WVMchs2QWo/doca-kernel-repo-24.10.1.1.4.0-1.kver.5.14.0.427.31.1.el9.4.x86.64.x86_64.rpm
+  changed_when: false
+
+- name: Create dnf cache
+  ansible.builtin.command: dnf makecache
+
+- name: Install DOCA repository package
+  ansible.builtin.dnf:
+    name: "{{ _doca_kernel_repo.stdout }}"
+    disable_gpg_check: true
+
+- name: Install DOCA packages
+  ansible.builtin.dnf:
+    name: "{{ doca_profile }}"
+
+- name: Cleanup DOCA build directories
+  ansible.builtin.file:
+    state: absent
+    path: "{{ (_doca_kernel_repo.stdout | split('/'))[:3] | join('/') }}" # leading / means 1st element of split list is ''
+
+- name: Update initramfs
+  ansible.builtin.command:
+    cmd: dracut -f
+  register: _doca_dracut
+  failed_when: _doca_dracut.stderr != '' # appears rc is always 0
+
+- name: Load the new driver
+  ansible.builtin.command: /etc/init.d/openibd restart
diff --git a/ansible/roles/doca/tasks/main.yml b/ansible/roles/doca/tasks/main.yml
new file mode 100644
index 000000000..e7a272f38
--- /dev/null
+++ b/ansible/roles/doca/tasks/main.yml
@@ -0,0 +1 @@
+- include_tasks: install.yml
diff --git a/ansible/roles/eessi/tasks/main.yaml b/ansible/roles/eessi/tasks/main.yaml
index d121b6fdd..c61625b0e 100644
--- a/ansible/roles/eessi/tasks/main.yaml
+++ b/ansible/roles/eessi/tasks/main.yaml
@@ -10,6 +10,7 @@
 - name: Add CVMFS repo
   dnf:
     name: https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm
+    disable_gpg_check: true
 
 - name: Install CVMFS
   dnf:
diff --git a/ansible/roles/k3s/README.md b/ansible/roles/k3s/README.md
new file mode 100644
index 000000000..68e8e2410
--- /dev/null
+++ b/ansible/roles/k3s/README.md
@@ -0,0 +1,16 @@
+k3s
+=====
+
+Installs k3s agent and server services on nodes and an ansible-init playbook to activate them. The service that each node will activate on init is determined by OpenStack metadata. Also includes Helm install. Currently only supports a single k3s-server
+(i.e one control node). Install based on the [official k3s ansible role](https://github.com/k3s-io/k3s-ansible).
+
+
+Requirements
+------------
+
+`azimuth_cloud.image_utils.linux_ansible_init` must have been run previously on targeted nodes during image build.
+
+Role Variables
+--------------
+
+- `k3s_version`: Optional str. K3s version to install, see [official releases](https://github.com/k3s-io/k3s/releases/).
diff --git a/ansible/roles/k3s/defaults/main.yml b/ansible/roles/k3s/defaults/main.yml
new file mode 100644
index 000000000..ba9a1a899
--- /dev/null
+++ b/ansible/roles/k3s/defaults/main.yml
@@ -0,0 +1,5 @@
+# Warning: changes to these variables won't be reflected in the cluster/image if k3s is already installed
+k3s_version: "v1.31.0+k3s1"
+k3s_selinux_release: v1.6.latest.1
+k3s_selinux_rpm_version: 1.6-1
+k3s_helm_version: v3.11.0
diff --git a/ansible/roles/k3s/files/start_k3s.yml b/ansible/roles/k3s/files/start_k3s.yml
new file mode 100644
index 000000000..b9b82f1c4
--- /dev/null
+++ b/ansible/roles/k3s/files/start_k3s.yml
@@ -0,0 +1,36 @@
+- hosts: localhost
+  become: true
+  vars:
+    os_metadata: "{{ lookup('url', 'http://169.254.169.254/openstack/latest/meta_data.json') | from_json }}"
+    k3s_token: "{{ os_metadata.meta.k3s_token }}"
+    k3s_server_name: "{{ os_metadata.meta.control_address }}"
+    service_name: "{{ 'k3s-agent' if k3s_server_name is defined else 'k3s' }}"
+  tasks:
+    - name: Ensure password directory exists
+      ansible.builtin.file: 
+        path: "/etc/rancher/node"
+        state: directory
+        
+    - name: Set agent node password as token # uses token to keep password consistent between reimages
+      ansible.builtin.copy:
+        dest: /etc/rancher/node/password
+        content: "{{ k3s_token }}"
+      
+    - name: Add the token for joining the cluster to the environment
+      no_log: true # avoid logging the server token
+      ansible.builtin.lineinfile:
+        path: "/etc/systemd/system/{{ service_name }}.service.env"
+        line: "K3S_TOKEN={{ k3s_token }}"
+
+    - name: Add server url to agents
+      ansible.builtin.lineinfile:
+        path: "/etc/systemd/system/{{ service_name }}.service.env"
+        line: "K3S_URL=https://{{ k3s_server_name }}:6443"
+      when: k3s_server_name is defined
+
+    - name: Start k3s service
+      ansible.builtin.systemd:
+        name: "{{ service_name }}"
+        daemon_reload: true
+        state: started
+        enabled: true
diff --git a/ansible/roles/k3s/tasks/install.yml b/ansible/roles/k3s/tasks/install.yml
new file mode 100644
index 000000000..77b95a509
--- /dev/null
+++ b/ansible/roles/k3s/tasks/install.yml
@@ -0,0 +1,78 @@
+---
+
+- name: Check for existing k3s installation
+  stat:
+    path: /var/lib/rancher/k3s
+  register: stat_result
+
+- name: Perform air-gapped installation of k3s
+  # Using air-gapped install so containers are pre-installed to avoid rate-limiting from registries on cluster startup
+  when: not stat_result.stat.exists
+  block:
+
+  - name: Download k3s binary
+    ansible.builtin.get_url:
+      url: "https://github.com/k3s-io/k3s/releases/download/{{ k3s_version | urlencode }}/k3s"
+      dest: /usr/bin/k3s
+      owner: root
+      group: root
+      mode: "0755"
+
+  - name: Install k3s SELinux policy package
+    yum:
+      name: "https://github.com/k3s-io/k3s-selinux/releases/download/{{ k3s_selinux_release }}/k3s-selinux-{{ k3s_selinux_rpm_version }}.el{{ ansible_distribution_major_version }}.noarch.rpm"
+      disable_gpg_check: true
+
+  - name: Create image directory
+    ansible.builtin.file: 
+      path: "/var/lib/rancher/k3s/agent/images"
+      state: directory
+
+  - name: Install k3s' internal images
+    ansible.builtin.get_url:
+      url: "https://github.com/k3s-io/k3s/releases/download/{{ k3s_version | urlencode }}/k3s-airgap-images-amd64.tar.zst"
+      dest: /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar.zst
+
+  - name: Download k3s install script
+    ansible.builtin.get_url:
+      url: https://get.k3s.io/
+      timeout: 120
+      dest: /usr/bin/k3s-install.sh
+      owner: root
+      group: root
+      mode: "0755"
+
+  - name: Install k3s
+    ansible.builtin.shell:
+      cmd: /usr/bin/k3s-install.sh
+    environment:
+      INSTALL_K3S_VERSION: "{{ k3s_version }}"
+      INSTALL_K3S_EXEC: "{{ item }}"
+      INSTALL_K3S_SKIP_START: "true"
+      INSTALL_K3S_SKIP_ENABLE: "true"
+      INSTALL_K3S_BIN_DIR: "/usr/bin"
+      INSTALL_K3S_SKIP_DOWNLOAD: "true"
+    changed_when: true
+    loop:
+      - server --disable=traefik
+      - agent
+
+- name: Install helm
+  unarchive:
+    src: "https://get.helm.sh/helm-{{ k3s_helm_version }}-linux-amd64.tar.gz"
+    dest: /usr/bin
+    extra_opts: "--strip-components=1"
+    owner: root
+    group: root
+    mode: 0755
+    remote_src: true
+
+- name: Add k3s kubeconfig as environment variable
+  ansible.builtin.lineinfile:
+    path: /etc/environment
+    line: "KUBECONFIG=/etc/rancher/k3s/k3s.yaml"
+
+- name: Install ansible-init playbook for k3s agent or server activation
+  copy:
+    src: start_k3s.yml
+    dest: /etc/ansible-init/playbooks/0-start-k3s.yml
diff --git a/ansible/roles/k9s/tasks/main.yml b/ansible/roles/k9s/tasks/main.yml
new file mode 100644
index 000000000..674b4dffb
--- /dev/null
+++ b/ansible/roles/k9s/tasks/main.yml
@@ -0,0 +1,44 @@
+---
+  
+  - name: Check if k9s is installed
+    ansible.builtin.stat:
+      path: "/usr/bin/k9s"
+    register: _k9s_stat_result
+
+  - name: Install k9s and clean up temporary files
+    block:
+    - name: Create install directory
+      ansible.builtin.file:
+        path: /tmp/k9s
+        state: directory
+        owner: root
+        group: root
+        mode: "744"
+      when: not _k9s_stat_result.stat.exists
+
+    - name: Download k9s
+      ansible.builtin.get_url:
+        url: https://github.com/derailed/k9s/releases/download/v0.32.5/k9s_Linux_amd64.tar.gz
+        dest: /tmp/k9s/k9s_Linux_amd64.tar.gz
+        owner: root
+        group: root
+        mode: "744"
+
+    - name: Unpack k9s binary
+      ansible.builtin.unarchive:
+        src: /tmp/k9s/k9s_Linux_amd64.tar.gz
+        dest: /tmp/k9s
+        remote_src: yes
+
+    - name: Add k9s to root path
+      ansible.builtin.copy:
+        src: /tmp/k9s/k9s
+        dest: /usr/bin/k9s
+        mode: u+rwx
+        remote_src: yes
+
+    - name: Cleanup k9s install directory
+      ansible.builtin.file:
+        path: /tmp/k9s
+        state: absent
+    when: not _k9s_stat_result.stat.exists
diff --git a/ansible/roles/lustre/README.md b/ansible/roles/lustre/README.md
new file mode 100644
index 000000000..3ba0dad56
--- /dev/null
+++ b/ansible/roles/lustre/README.md
@@ -0,0 +1,27 @@
+# lustre
+
+Install and configure a Lustre client. This builds RPM packages from source.
+
+**NB:** The `install.yml` playbook in this role should only be run during image build and is not idempotent. This will install the `kernel-devel` package; if not already installed (e.g. from an `ofed` installation), this may require enabling update of DNF packages during build using `update_enable=true`, which will upgrade the kernel as well.
+
+**NB:** Currently this only supports RockyLinux 9.
+
+## Role Variables
+
+- `lustre_version`: Optional str. Version of lustre to build, default `2.15.6` which is the first version with EL9.5 support
+- `lustre_lnet_label`: Optional str. The "lnet label" part of the host's NID, e.g. `tcp0`. Only the `tcp` protocol type is currently supported. Default `tcp`.
+- `lustre_mgs_nid`: Required str. The NID(s) for the MGS, e.g. `192.168.227.11@tcp1` (separate mutiple MGS NIDs using `:`).
+- `lustre_mounts`: Required list. Define Lustre filesystems and mountpoints as a list of dicts with keys:
+    - `fs_name`: Required str. The name of the filesystem to mount
+    - `mount_point`: Required str. Path to mount filesystem at.
+    - `mount_state`: Optional mount state, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `lustre_mount_state`.
+    - `mount_options`: Optional mount options. Default is `lustre_mount_options`.
+- `lustre_mount_state`. Optional default mount state for all mounts, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `mounted`.
+- `lustre_mount_options`. Optional default mount options. Default values are systemd defaults from [Lustre client docs](http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes).
+
+The following variables control the package build and and install and should not generally be required:
+- `lustre_build_packages`: Optional list. Prerequisite packages required to build Lustre. See `defaults/main.yml`.
+- `lustre_build_dir`: Optional str. Path to build lustre at, default `/tmp/lustre-release`.
+- `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellanox OFED, without support for GSS keys.
+- `lustre_rpm_globs`: Optional list. Shell glob patterns for rpms to install. Note order is important as the built RPMs are not in a yum repo. Default is just the `kmod-lustre-client` and `lustre-client` packages.
+- `lustre_build_cleanup`: Optional bool. Whether to uninstall prerequisite packages and delete the build directories etc. Default `true`.
diff --git a/ansible/roles/lustre/defaults/main.yml b/ansible/roles/lustre/defaults/main.yml
new file mode 100644
index 000000000..40389970c
--- /dev/null
+++ b/ansible/roles/lustre/defaults/main.yml
@@ -0,0 +1,36 @@
+lustre_version: '2.15.6' # https://www.lustre.org/lustre-2-15-6-released/
+lustre_lnet_label: tcp
+#lustre_mgs_nid:
+lustre_mounts: []
+lustre_mount_state: mounted
+lustre_mount_options: 'defaults,_netdev,noauto,x-systemd.automount,x-systemd.requires=lnet.service'
+
+# below variables are for build and should not generally require changes
+lustre_build_packages:
+  - "kernel-devel-{{ ansible_kernel }}"
+  - git
+  - gcc
+  - libtool
+  - python3
+  - python3-devel
+  - openmpi
+  - elfutils-libelf-devel
+  - libmount-devel
+  - libnl3-devel
+  - libyaml-devel
+  - rpm-build
+  - kernel-abi-stablelists
+  - libaio
+  - libaio-devel
+lustre_build_dir: /tmp/lustre-release
+lustre_configure_opts:
+  - --disable-server
+  - --with-linux=/usr/src/kernels/*
+  - --with-o2ib=/usr/src/ofa_kernel/default
+  - --disable-maintainer-mode
+  - --disable-gss-keyring
+  - --enable-mpitests=no
+lustre_rpm_globs: # NB: order is important here, as not installing from a repo
+  - "kmod-lustre-client-{{ lustre_version | split('.') | first }}*" # only take part of the version as -RC versions produce _RC rpms
+  - "lustre-client-{{ lustre_version | split('.') | first }}*"
+lustre_build_cleanup: true
diff --git a/ansible/roles/lustre/tasks/configure.yml b/ansible/roles/lustre/tasks/configure.yml
new file mode 100644
index 000000000..b77e02ed9
--- /dev/null
+++ b/ansible/roles/lustre/tasks/configure.yml
@@ -0,0 +1,47 @@
+- name: Gather Lustre interface info
+  shell:
+    cmd: |
+      ip r get {{ _lustre_mgs_ip }}
+  changed_when: false
+  register: _lustre_ip_r_mgs
+  vars:
+    _lustre_mgs_ip: "{{ lustre_mgs_nid | split('@') | first }}"
+
+- name: Set facts for Lustre interface
+  set_fact:
+    _lustre_interface: "{{ _lustre_ip_r_mgs_info[4] }}"
+    _lustre_ip: "{{ _lustre_ip_r_mgs_info[6] }}"
+  vars:
+    _lustre_ip_r_mgs_info: "{{ _lustre_ip_r_mgs.stdout_lines.0 | split }}"
+    # first line e.g. "10.167.128.1 via 10.179.0.2 dev eth0 src 10.179.3.149 uid 1000"
+
+- name: Write LNet configuration file
+  template:
+    src: lnet.conf.j2
+    dest:  /etc/lnet.conf # exists from package install, expected by lnet service
+    owner: root
+    group: root
+    mode: u=rw,go=r # from package install
+  register: _lnet_conf
+
+- name: Ensure lnet service state
+  systemd:
+    name: lnet
+    state: "{{ 'restarted' if _lnet_conf.changed else 'started' }}"
+
+- name: Ensure mount points exist
+  ansible.builtin.file:
+    path: "{{ item.mount_point }}"
+    state: directory
+  loop: "{{ lustre_mounts }}"
+  when: "(item.mount_state | default(lustre_mount_state)) != 'absent'"
+
+- name: Mount lustre filesystem
+  ansible.posix.mount:
+    fstype: lustre
+    src: "{{ lustre_mgs_nid }}:/{{ item.fs_name }}"
+    path: "{{ item.mount_point }}"
+    state: "{{ (item.mount_state | default(lustre_mount_state)) }}"
+    opts: "{{ item.mount_options | default(lustre_mount_options) }}"
+  loop: "{{ lustre_mounts }}"
+ 
\ No newline at end of file
diff --git a/ansible/roles/lustre/tasks/install.yml b/ansible/roles/lustre/tasks/install.yml
new file mode 100644
index 000000000..852b4652f
--- /dev/null
+++ b/ansible/roles/lustre/tasks/install.yml
@@ -0,0 +1,49 @@
+- name: Install lustre build prerequisites
+  ansible.builtin.dnf:
+    name: "{{ lustre_build_packages }}"
+  register: _lustre_dnf_build_packages
+ 
+- name: Clone lustre git repo
+  # https://git.whamcloud.com/?p=fs/lustre-release.git;a=summary
+  ansible.builtin.git:
+    repo: git://git.whamcloud.com/fs/lustre-release.git
+    dest: "{{ lustre_build_dir }}"
+    version: "{{ lustre_version }}"
+
+- name: Prepare for lustre configuration
+  ansible.builtin.command:
+    cmd: sh ./autogen.sh
+    chdir: "{{ lustre_build_dir }}"
+
+- name: Configure lustre build
+  ansible.builtin.command:
+    cmd: "./configure {{ lustre_configure_opts | join(' ') }}"
+    chdir: "{{ lustre_build_dir }}"
+
+- name: Build lustre
+  ansible.builtin.command:
+    cmd: make rpms
+    chdir: "{{ lustre_build_dir }}"
+
+- name: Find rpms
+  ansible.builtin.find:
+    paths: "{{ lustre_build_dir }}"
+    patterns: "{{ lustre_rpm_globs }}"
+    use_regex: false
+  register: _lustre_find_rpms
+
+- name: Check rpms found
+  assert:
+    that: _lustre_find_rpms.files | length
+    fail_msg: "No lustre repos found with lustre_rpm_globs = {{ lustre_rpm_globs }}"
+
+- name: Install lustre rpms
+  ansible.builtin.dnf:
+    name: "{{ _lustre_find_rpms.files | map(attribute='path')}}"
+    disable_gpg_check: yes
+
+- name: Delete lustre build dir
+  file:
+    path: "{{ lustre_build_dir }}"
+    state: absent
+  when: lustre_build_cleanup | bool
diff --git a/ansible/roles/lustre/tasks/validate.yml b/ansible/roles/lustre/tasks/validate.yml
new file mode 100644
index 000000000..fe65a4d1a
--- /dev/null
+++ b/ansible/roles/lustre/tasks/validate.yml
@@ -0,0 +1,27 @@
+- name: Assert using RockyLinux 9
+  assert:
+    that: ansible_distribution_major_version | int == 9
+    fail_msg: The 'lustre' role requires RockyLinux 9
+
+- name: Check kernel-devel package is installed
+  command: "dnf list --installed kernel-devel-{{ ansible_kernel }}"
+  changed_when: false
+  # NB: we don't check here the kernel will remain the same after reboot etc, see ofed/install.yml
+
+- name: Ensure SELinux in permissive mode
+  assert:
+    that: selinux_state in ['permissive', 'disabled']
+    fail_msg: "SELinux must be permissive for Lustre not '{{ selinux_state }}'; see variable selinux_state"
+
+- name: Ensure lustre_mgs_nid is defined
+  assert:
+    that: lustre_mgs_nid is defined
+    fail_msg: Variable lustre_mgs_nid must be defined
+
+- name: Ensure lustre_mounts entries define filesystem name and mount point
+  assert:
+    that:
+      - item.fs_name is defined
+      - item.mount_point is defined
+    fail_msg: All lustre_mounts entries must specify fs_name and mount_point
+  loop: "{{ lustre_mounts }}"
diff --git a/ansible/roles/lustre/templates/lnet.conf.j2 b/ansible/roles/lustre/templates/lnet.conf.j2
new file mode 100644
index 000000000..363308e32
--- /dev/null
+++ b/ansible/roles/lustre/templates/lnet.conf.j2
@@ -0,0 +1,6 @@
+net:
+    - net type: {{ lustre_lnet_label }}
+      local NI(s):
+        - nid: {{ _lustre_ip }}@{{ lustre_lnet_label }}
+          interfaces:
+              0: {{ _lustre_interface }}
diff --git a/ansible/roles/openondemand/README.md b/ansible/roles/openondemand/README.md
index c6a4f3f9f..365265df0 100644
--- a/ansible/roles/openondemand/README.md
+++ b/ansible/roles/openondemand/README.md
@@ -17,7 +17,7 @@ This uses the [osc.ood](https://github.com/OSC/ood-ansible) Ansible role to prov
 ### General
 
 - `openondemand_clusters`: Required. Synonym for [osc.ood: clusters](https://github.com/OSC/ood-ansible#clusters) role variable.
-- `openondemand_servername`: Required. Synonym for [osc.ood: servername](https://github.com/OSC/ood-ansible/blob/master/defaults/main/ood_portal.yml#L27) role variable. This defines what the Open Ondemand portal's Apache server uses for the [name-based virtual host](https://httpd.apache.org/docs/current/mod/core.html#servername). It should be the IP or hostname(+domain) part of the URL used to access Open Ondemand in the browser, e.g. `ondemand.mysite.org`. **NB:** If a domain or external IP is not available, specify the host's internal IP here and use ssh with a `DynamicForward` option and a SOCKS proxy to access this address. Using ssh's `LocalForward` option is not recommended as the server name will have to be `localhost` which causes some issues.
+- `openondemand_servername`: Required. Synonym for [osc.ood: servername](https://github.com/OSC/ood-ansible/blob/master/defaults/main/ood_portal.yml#L27) role variable. This defines what the Open Ondemand portal's Apache server uses for the [name-based virtual host](https://httpd.apache.org/docs/current/mod/core.html#servername). It should be the IP or hostname(+domain) part of the URL used to access Open Ondemand in the browser, e.g. `ondemand.mysite.org`. **NB:** If a domain or external IP is not available, specify the host's internal IP here and use ssh with a `DynamicForward` option and a SOCKS proxy to access this address. Using ssh's `LocalForward` option is not recommended as the server name will have to be `localhost` which causes some issues. Changing this value on an already deployed cluster requires a reboot of the login node for OOD app state to be correctly refreshed.
 
 ### Authentication
 See the Open Ondemand [Authentication docs](https://osc.github.io/ood-documentation/latest/authentication/overview.html) for an overview of the authentication process.
@@ -77,7 +77,7 @@ The Open Ondemand portal can proxy other servers. Variables:
   to proxy:
   - All "compute" nodes, e.g. for Open Ondemand interactive apps such as remote desktop and Jupyter notebook server.
   - The Grafana server - note a link to Grafana is always added to the Open Ondemand dashboard.
-  
+
   The exact pattern depends on inventory hostnames / partitions / addresses.
 
 - `openondemand_node_proxy_directives`: Optional, default ''. Multiline string to insert into Apache directives definition for `node_uri` ([docs](https://osc.github.io/ood-documentation/master/reference/files/ood-portal-yml.html#configure-reverse-proxy)).
diff --git a/ansible/roles/openondemand/tasks/main.yml b/ansible/roles/openondemand/tasks/main.yml
index 86184f13c..bd5706ecb 100644
--- a/ansible/roles/openondemand/tasks/main.yml
+++ b/ansible/roles/openondemand/tasks/main.yml
@@ -6,12 +6,19 @@
   loop: "{{ openondemand_osc_ood_defaults | dict2items }}"
   when: (item.key in hostvars[inventory_hostname]) or (item.value)
 
+# osc.ood variables are exposed to play here instead of setting 'public' in include role so that they will still be exposed during runtime
+- ansible.builtin.include_vars:
+    dir: "{{ playbook_dir }}/roles/osc.ood/defaults/main"
+
+- ansible.builtin.include_vars:
+    file: "{{ playbook_dir }}/roles/osc.ood/vars/Rocky/{{ ansible_distribution_major_version }}.yml"
+
 # if using PAM auth we need apache installed but NOT started so split the osc.ood role up:
 - include_role:
     name: osc.ood
     tasks_from: install-package.yml
     vars_from: "Rocky/{{ ansible_distribution_major_version }}.yml"
-    public: yes # Expose the vars from this role to the rest of the play
+  when: appliances_mode != 'configure'
   # can't set vars: from a dict hence the workaround above
 
 - include_tasks:
diff --git a/ansible/roles/openondemand/tasks/pam_auth.yml b/ansible/roles/openondemand/tasks/pam_auth.yml
index 0edce622f..6bc4bda36 100644
--- a/ansible/roles/openondemand/tasks/pam_auth.yml
+++ b/ansible/roles/openondemand/tasks/pam_auth.yml
@@ -1,6 +1,6 @@
 # https://osc.github.io/ood-documentation/latest/authentication/pam.html
 ---
-- name: Install Apache PAM module
+- name: Install Apache PAM module # Extracted from start of roles/openondemand/tasks/pam_auth.yml to ensure only installed during build
   yum:
     name: mod_authnz_pam
 
diff --git a/ansible/roles/openondemand/tasks/vnc_compute.yml b/ansible/roles/openondemand/tasks/vnc_compute.yml
index 388e3b3c5..6ec340249 100644
--- a/ansible/roles/openondemand/tasks/vnc_compute.yml
+++ b/ansible/roles/openondemand/tasks/vnc_compute.yml
@@ -48,6 +48,7 @@
   tags: install
   yum:
     name: '@Xfce'
+  when: appliances_mode != 'configure' # dnf group/module installs aren't idempotent so only run during build
   
 # - name: Ensure python3.9 installed
 #   dnf:
diff --git a/ansible/roles/passwords/defaults/main.yml b/ansible/roles/passwords/defaults/main.yml
index 55680ae37..929aac465 100644
--- a/ansible/roles/passwords/defaults/main.yml
+++ b/ansible/roles/passwords/defaults/main.yml
@@ -8,6 +8,9 @@ slurm_appliance_secrets:
   vault_openhpc_mungekey: "{{ secrets_openhpc_mungekey | default(vault_openhpc_mungekey | default(secrets_openhpc_mungekey_default)) }}"
   vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}"
   vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}"
+  vault_k3s_token: "{{ vault_k3s_token | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
+  vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}"
+  vault_demo_user_password: "{{ vault_demo_user_password | default(lookup('password', '/dev/null')) }}"
 
 secrets_openhpc_mungekey_default:
   content: "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') }}"
diff --git a/ansible/roles/passwords/tasks/main.yml b/ansible/roles/passwords/tasks/main.yml
index 21b10f780..d67dc84ea 100644
--- a/ansible/roles/passwords/tasks/main.yml
+++ b/ansible/roles/passwords/tasks/main.yml
@@ -7,14 +7,14 @@
   delegate_to: localhost
   run_once: true
 
-# - name: Ensure munge key directory exists
-#   file:
-#     state: directory
-#     recurse: true
-#     path: "{{ openhpc_passwords_mungekey_output_path | dirname }}"
+- name: Get templated passwords from target environment
+# inventory group/host vars created in a play cannot be accessed in the same play, even after meta: refresh_inventory
+  ansible.builtin.include_vars:
+    file: "{{ openhpc_passwords_output_path }}"
 
-# - name: Create a munge key
-#   copy:
-#     content:  "{{ lookup('password', '/dev/null chars=ascii_letters,digits,hexdigits,punctuation') }}"
-#     dest: "{{ openhpc_passwords_mungekey_output_path }}"
-#     force: false
\ No newline at end of file
+- name:  Template k3s token to terraform
+  template:
+    src: k3s-token.auto.tfvars.json.j2
+    dest: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/terraform/k3s-token.auto.tfvars.json"
+  delegate_to: localhost
+  run_once: true
diff --git a/ansible/roles/passwords/tasks/validate.yml b/ansible/roles/passwords/tasks/validate.yml
index 9279ffdbf..b30b0696e 100644
--- a/ansible/roles/passwords/tasks/validate.yml
+++ b/ansible/roles/passwords/tasks/validate.yml
@@ -1,4 +1,4 @@
 - name: Assert secrets created
   assert:
-    that: (hostvars[inventory_hostname].keys() | select('contains', 'vault_') | length) > 1 # 1 as may have vault_testuser_password defined in dev
+    that: (hostvars[inventory_hostname].keys() | select('contains', 'vault_') | length) > 1 # 1 as may have vault_demo_user_password defined in dev
     fail_msg: "No inventory variables 'vault_*' found: Has ansible/adhoc/generate-passwords.yml been run?"
diff --git a/ansible/roles/passwords/templates/k3s-token.auto.tfvars.json.j2 b/ansible/roles/passwords/templates/k3s-token.auto.tfvars.json.j2
new file mode 100644
index 000000000..2a8fabba8
--- /dev/null
+++ b/ansible/roles/passwords/templates/k3s-token.auto.tfvars.json.j2
@@ -0,0 +1,3 @@
+{
+    "k3s_token": "{{ vault_k3s_token }}"
+}
\ No newline at end of file
diff --git a/ansible/roles/persist_hostkeys/README.md b/ansible/roles/persist_hostkeys/README.md
index 2d823dc36..6201a104b 100644
--- a/ansible/roles/persist_hostkeys/README.md
+++ b/ansible/roles/persist_hostkeys/README.md
@@ -1,8 +1,5 @@
 # persist_hostkeys
 
-Save hostkeys to persistent storage and restore them after a rebuild/reimage.
+Idempotently generates a persistent set of hostkeys and restores them after a rebuild/reimage.
 
-Add hosts to the `persist_hostkeys` group to enable.
-
-This role has no variables but hosts in this group must have `appliances_state_dir`
-defined as a directory they can write to on persistent storage.
+Add hosts to the `persist_hostkeys` group to enable. All hosts in group will share the same set hostkeys.
diff --git a/ansible/roles/persist_hostkeys/defaults/main.yml b/ansible/roles/persist_hostkeys/defaults/main.yml
new file mode 100644
index 000000000..3c0000466
--- /dev/null
+++ b/ansible/roles/persist_hostkeys/defaults/main.yml
@@ -0,0 +1,2 @@
+persist_hostkeys_state_server: "{{ groups['control'] | first }}"
+persist_hostkeys_state_dir: "{{ hostvars[persist_hostkeys_state_server]['appliances_state_dir'] }}/hostkeys"
diff --git a/ansible/roles/persist_hostkeys/tasks/main.yml b/ansible/roles/persist_hostkeys/tasks/main.yml
index 47493220d..deff112f7 100644
--- a/ansible/roles/persist_hostkeys/tasks/main.yml
+++ b/ansible/roles/persist_hostkeys/tasks/main.yml
@@ -1,33 +1,47 @@
 ---
 
-- name: Ensure hostkeys directory exists on persistent storage
-  file:
-    path: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}"
-    state: directory
-    owner: root
-    group: root
-    mode: 0600
+- name: Generate persistent hostkeys in state directory
+  delegate_to: "{{ persist_hostkeys_state_server }}"
+  block:
+  - name: Ensure hostkeys directory exists on persistent storage
+    file:
+      path: "{{ persist_hostkeys_state_dir }}"
+      state: directory
+      owner: root
+      group: root
+      mode: 0600
 
-- name: Copy hostkeys from persistent storage
-  # won't fail if no keys are in persistent storage
-  copy:
-    src: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}/"
-    dest: /etc/ssh/
-    remote_src: true
+  - name: Check for existing hostkeys
+    find:
+      paths: "{{ persist_hostkeys_state_dir }}/"
+    register: _files_found
+
+  - name: Generate hostkeys
+    when: _files_found.matched == 0
+    shell:
+    # ssh-keygen -A needs a directory with an /etc/ssh suffix to write hostkeys into
+      cmd: |
+        mkdir -p {{ persist_hostkeys_state_dir }}/etc/ssh
+        ssh-keygen -A -N '' -f {{ persist_hostkeys_state_dir }}
+        mv {{ persist_hostkeys_state_dir }}/etc/ssh/* {{ persist_hostkeys_state_dir }}
+        rm -rf {{ persist_hostkeys_state_dir }}/etc/ssh
+  
+  - name: Get created key names
+    find:
+      path: "{{ persist_hostkeys_state_dir }}/"
+    register: _find_ssh_keys
 
-- name: Find hostkeys
-  find:
-    path: /etc/ssh/
-    patterns: ssh_host_*_key*
-  register: _find_ssh_keys
+  - name: Create in-memory copies of keys
+    ansible.builtin.slurp:
+      src: "{{ item.path }}"
+    loop: "{{ _find_ssh_keys.files }}"
+    register: _slurp_keys
 
-- name: Persist hostkeys
+- name: Copy keys to hosts
+  no_log: true
   copy:
-    dest: "{{ appliances_state_dir }}/hostkeys/{{ inventory_hostname }}/"
-    src: "{{ item }}"
-    remote_src: true
-    mode: preserve
-  loop: "{{ _find_ssh_keys.files | map(attribute='path') }}"
+    content: "{{ item.content | b64decode }}"
+    dest: "/etc/ssh/{{ item.source | regex_search('[^/]+$') }}"
+  loop: "{{ _slurp_keys.results }}"
 
 - meta: reset_connection
-
diff --git a/ansible/roles/podman/tasks/config.yml b/ansible/roles/podman/tasks/config.yml
index 5fea3c2e0..74cf1d576 100644
--- a/ansible/roles/podman/tasks/config.yml
+++ b/ansible/roles/podman/tasks/config.yml
@@ -55,6 +55,7 @@
       # Type Path Mode User Group Age Argument
       R! /tmp/containers-user-*
       R! /tmp/podman-run-*
+      R! /tmp/storage-run-*
     dest: /etc/tmpfiles.d/podman-local.conf
     owner: root
     group: root
diff --git a/ansible/roles/pulp_site/.gitignore b/ansible/roles/pulp_site/.gitignore
new file mode 100644
index 000000000..6738e49c1
--- /dev/null
+++ b/ansible/roles/pulp_site/.gitignore
@@ -0,0 +1 @@
+filter_plugins/__pycache__
\ No newline at end of file
diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml
new file mode 100644
index 000000000..c549dac53
--- /dev/null
+++ b/ansible/roles/pulp_site/defaults/main.yml
@@ -0,0 +1,42 @@
+pulp_site_url: "{{ appliances_pulp_url }}"
+pulp_site_port: 8080
+pulp_site_username: admin # shouldn't be changed
+pulp_site_password: "{{ vault_pulp_admin_password }}"
+pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content
+pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os"
+pulp_site_validate_certs: false
+pulp_site_install_dir: '/home/rocky/pulp'
+pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}"
+pulp_site_target_facts: "{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}"
+pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}"
+pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}"
+
+pulp_site_rpm_info:
+- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
+- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
+- name: "crb-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
+- name: "extras-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
+- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+- name: "ohpc-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+- name: "ohpc-updates-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+- name: "ceph-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+
+pulp_site_rpm_repo_defaults:
+  remote_username: "{{ pulp_site_upstream_username }}"
+  remote_password: "{{ pulp_site_upstream_password }}"
+  policy: on_demand
+  state: present
+
+_pulp_site_rpm_info_all: "{{ pulp_site_rpm_info | map('combine', pulp_site_rpm_repo_defaults) }}"
+
+pulp_site_rpm_repos: "{{ _pulp_site_rpm_info_all | to_rpm_repos(pulp_site_upstream_content_url) }}"
+pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}"
+pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}"
diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py
new file mode 100644
index 000000000..50e912685
--- /dev/null
+++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py
@@ -0,0 +1,31 @@
+class FilterModule(object):
+    def filters(self):
+        return {
+            'to_rpm_repos': self.to_rpm_repos,
+            'to_rpm_pubs': self.to_rpm_pubs,
+            'to_rpm_distros': self.to_rpm_distros
+        }
+
+    def to_rpm_repos(self, list, pulp_url):
+        repo_list = map(lambda x: {
+            'name': x['name'],
+            'url': pulp_url+'/'+x['subpath'],
+            'remote_username': x['remote_username'],
+            'remote_password': x['remote_password'],
+            'policy': x['policy'],
+            'state': x['state'] }, list)
+        return repo_list
+    
+    def to_rpm_pubs(self, list):
+        pub_list = map(lambda x: {
+            'repository': x['name'],
+            'state': x['state'] }, list)
+        return pub_list
+    
+    def to_rpm_distros(self, list):
+        distro_list = map(lambda x: {
+            'name': x['name'],
+            'repository': x['name'],
+            'base_path': x['subpath'],
+            'state': x['state'] }, list)
+        return distro_list
\ No newline at end of file
diff --git a/ansible/roles/pulp_site/tasks/install.yml b/ansible/roles/pulp_site/tasks/install.yml
new file mode 100644
index 000000000..39b4fcd97
--- /dev/null
+++ b/ansible/roles/pulp_site/tasks/install.yml
@@ -0,0 +1,43 @@
+---
+
+- name: Install packages
+  dnf:
+    name:
+    - podman
+
+- name: Create install directories
+  ansible.builtin.file:
+    state: directory
+    path: "{{ pulp_site_install_dir }}/{{ item }}"
+  loop:
+  - settings/certs
+  - pulp_storage
+  - pgsql
+  - containers
+
+- name: Template settings file
+  ansible.builtin.template:
+    src: settings.py.j2
+    dest: "{{ pulp_site_install_dir }}/settings/settings.py"
+
+- name: Install pulp podman container
+  containers.podman.podman_container:
+    name: pulp
+    publish:
+      - "{{ pulp_site_port }}:80"
+    volume:
+    - "{{ pulp_site_install_dir }}/settings:/etc/pulp{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/pulp_storage:/var/lib/pulp{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/pgsql:/var/lib/pgsql{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ pulp_site_selinux_suffix }}"
+    device: /dev/fuse
+    image: docker.io/pulp/pulp:3.68.1
+
+- name: Reset admin password once container has initialised
+  no_log: true
+  ansible.builtin.shell:
+    cmd: "podman exec pulp bash -c 'pulpcore-manager reset-admin-password -p {{ pulp_site_password }}'"
+  register: _admin_reset_output
+  until: 0 == _admin_reset_output.rc
+  retries: 6
+  delay: 30
diff --git a/ansible/roles/pulp_site/tasks/sync.yml b/ansible/roles/pulp_site/tasks/sync.yml
new file mode 100644
index 000000000..5ef2bc5f1
--- /dev/null
+++ b/ansible/roles/pulp_site/tasks/sync.yml
@@ -0,0 +1,78 @@
+---
+
+- ansible.builtin.assert:
+    that: pulp_site_upstream_password != ''
+    quiet: true
+    fail_msg: "Upstream password not set. Either set env var ARK_PASSWORD or override pulp_site_upstream_password."
+
+- name: Wait for Pulp server
+  pulp.squeezer.status:
+    pulp_url: "{{ pulp_site_url }}"
+    username: "{{ pulp_site_username }}"
+    password: "{{ pulp_site_password }}"
+  register: _pulp_status
+  until: _pulp_status.failed == false
+  retries: 30
+  delay: 20
+
+- name: Ensure Pulp CLI config directory exists
+  ansible.builtin.file:
+    path: ~/.config/pulp
+    state: directory
+
+- name: Create config file
+  no_log: true
+  ansible.builtin.template:
+    src: cli.toml.j2
+    dest: ~/.config/pulp/cli.toml
+    mode: '0644'
+
+- block:
+  - name: Ensure squeezer cache exists
+    ansible.builtin.file:
+      path: "{{ _cache_dir }}"
+      state: directory
+
+  - name: Check if squeezer cache is populated
+    ansible.builtin.stat:
+      path: "{{ _cache_dir }}/api.json"
+    register: _cache_stat
+
+  - name: Prepopulate squeezer cache # workaround for race on the cache
+    ansible.builtin.get_url:
+      url: "{{ pulp_site_url }}/pulp/api/v3/docs/api.json"
+      dest: "{{ _cache_dir }}/api.json"
+      timeout: 40
+    when: not _cache_stat.stat.exists
+  vars:
+    _cache_dir: "~/.cache/squeezer/{{ pulp_site_url | regex_replace( ':|/' , '_' ) }}"
+
+- name: Get Pulp repos from release train
+  ansible.builtin.include_role:
+    name: stackhpc.pulp.pulp_repository
+    tasks_from: rpm.yml
+  vars:
+    pulp_url: "{{ pulp_site_url }}"
+    pulp_username: "{{ pulp_site_username }}"
+    pulp_password: "{{ pulp_site_password }}"
+    pulp_repository_rpm_repos: "{{ pulp_site_rpm_repos }}"
+
+- name: Create Pulp publications
+  ansible.builtin.include_role:
+    name: stackhpc.pulp.pulp_publication
+    tasks_from: rpm.yml
+  vars:
+    pulp_url: "{{ pulp_site_url }}"
+    pulp_username: "{{ pulp_site_username }}"
+    pulp_password: "{{ pulp_site_password }}"
+    pulp_publication_rpm: "{{ pulp_site_rpm_publications }}"
+
+- name: Create Pulp distributions
+  ansible.builtin.include_role:
+    name: stackhpc.pulp.pulp_distribution
+    tasks_from: rpm.yml
+  vars:
+    pulp_url: "{{ pulp_site_url }}"
+    pulp_username: "{{ pulp_site_username }}"
+    pulp_password: "{{ pulp_site_password }}"
+    pulp_distribution_rpm: "{{ pulp_site_rpm_distributions }}"
diff --git a/ansible/roles/pulp_site/templates/cli.toml.j2 b/ansible/roles/pulp_site/templates/cli.toml.j2
new file mode 100644
index 000000000..06867902f
--- /dev/null
+++ b/ansible/roles/pulp_site/templates/cli.toml.j2
@@ -0,0 +1,14 @@
+[cli]
+base_url = "{{ pulp_site_url }}"
+username = "{{ pulp_site_username }}"
+password = "{{ pulp_site_password }}"
+api_root = "/pulp/"
+domain = "default"
+headers = []
+cert = ""
+key = ""
+verify_ssl = true
+format = "json"
+dry_run = false
+timeout = 0
+verbose = 0
diff --git a/ansible/roles/pulp_site/templates/settings.py.j2 b/ansible/roles/pulp_site/templates/settings.py.j2
new file mode 100644
index 000000000..200212e2c
--- /dev/null
+++ b/ansible/roles/pulp_site/templates/settings.py.j2
@@ -0,0 +1,2 @@
+CONTENT_ORIGIN='http://{{ ansible_fqdn }}:{{ pulp_site_port }}'
+TOKEN_AUTH_DISABLED=True
diff --git a/ansible/roles/sshd/README.md b/ansible/roles/sshd/README.md
new file mode 100644
index 000000000..0fac1d189
--- /dev/null
+++ b/ansible/roles/sshd/README.md
@@ -0,0 +1,9 @@
+# sshd
+
+Configure sshd.
+
+## Role variables
+
+- `sshd_password_authentication`: Optional bool. Whether to enable password login. Default `false`.
+- `sshd_conf_src`: Optional string. Path to sshd configuration template. Default is in-role template.
+- `sshd_conf_dest`: Optional string. Path to destination for sshd configuration file. Default is `/etc/ssh/sshd_config.d/10-ansible.conf` which overides `50-{cloud-init,redhat}` files, if present.
diff --git a/ansible/roles/sshd/defaults/main.yml b/ansible/roles/sshd/defaults/main.yml
new file mode 100644
index 000000000..672305799
--- /dev/null
+++ b/ansible/roles/sshd/defaults/main.yml
@@ -0,0 +1,3 @@
+sshd_password_authentication: false
+sshd_conf_src: sshd.conf.j2
+sshd_conf_dest: /etc/ssh/sshd_config.d/10-ansible.conf
diff --git a/ansible/roles/sshd/handlers/main.yml b/ansible/roles/sshd/handlers/main.yml
new file mode 100644
index 000000000..e11aa7801
--- /dev/null
+++ b/ansible/roles/sshd/handlers/main.yml
@@ -0,0 +1,4 @@
+- name: Restart sshd
+  systemd:
+    name: sshd
+    state: restarted
diff --git a/ansible/roles/sshd/tasks/configure.yml b/ansible/roles/sshd/tasks/configure.yml
new file mode 100644
index 000000000..8aafb5c19
--- /dev/null
+++ b/ansible/roles/sshd/tasks/configure.yml
@@ -0,0 +1,15 @@
+- name: Template sshd configuration
+  # NB: If parameters are defined multiple times the first value wins;
+  # The default /etc/ssh/sshd_config has
+  #   Include /etc/ssh/sshd_config.d/*.conf
+  # early on, which is generally held to be the correct approach, so adding
+  # values to the end of that file won't work
+  template:
+    src: "{{ sshd_conf_src }}"
+    dest: "{{ sshd_conf_dest }}"
+    owner: root
+    group: root
+    mode: u=rw,go=
+    validate: sshd -t -f %s
+  notify:
+    - Restart sshd
diff --git a/ansible/roles/sshd/tasks/main.yml b/ansible/roles/sshd/tasks/main.yml
new file mode 100644
index 000000000..84f493457
--- /dev/null
+++ b/ansible/roles/sshd/tasks/main.yml
@@ -0,0 +1 @@
+- import_tasks: configure.yml
diff --git a/ansible/roles/sshd/templates/sshd.conf.j2 b/ansible/roles/sshd/templates/sshd.conf.j2
new file mode 100644
index 000000000..2746f0642
--- /dev/null
+++ b/ansible/roles/sshd/templates/sshd.conf.j2
@@ -0,0 +1,2 @@
+# {{ ansible_managed }}
+PasswordAuthentication {{ 'yes' if sshd_password_authentication | bool else 'no' }}
diff --git a/ansible/roles/sssd/README.md b/ansible/roles/sssd/README.md
new file mode 100644
index 000000000..da4e63f31
--- /dev/null
+++ b/ansible/roles/sssd/README.md
@@ -0,0 +1,18 @@
+# sssd
+
+Install and configure [sssd](https://sssd.io/docs/introduction.html).
+
+
+## Role variables
+
+The only required configuration is to create a [sssd.conf](https://www.mankier.com/5/sssd.conf) template at the location specified by `sssd_conf_src`.
+
+- `sssd_packages`: Optional list. Packages to install.
+- `sssd_ldap_install`: Optional bool. Whether to install packages enabling SSSD to authenticate against LDAP. Default `false`.
+- `sssd_ldap_packages`: Optional list. Packages to install when using `sssd_ldap_install`.
+- `sssd_enable_mkhomedir`: Optional bool. Whether to enable creation of home directories on login. Default `false`.
+- `sssd_mkhomedir_packages`: Optional list. Packages to install when using `sssd_enable_mkhomedir`.
+- `sssd_conf_src`: Optional string. Path to `sssd.conf` template. Default (which must be created) is `{{ appliances_environment_root }}/files/sssd.conf.j2`.
+- `sssd_conf_dest`: Optional string. Path to destination for `sssd.conf`. Default `/etc/sssd/sssd.conf`.
+- `sssd_started`: Optional bool. Whether `sssd` service should be started.
+- `sssd_enabled`: Optional bool. Whether `sssd` service should be enabled.
diff --git a/ansible/roles/sssd/defaults/main.yml b/ansible/roles/sssd/defaults/main.yml
new file mode 100644
index 000000000..5bc58c990
--- /dev/null
+++ b/ansible/roles/sssd/defaults/main.yml
@@ -0,0 +1,12 @@
+sssd_packages:
+  - sssd-common
+sssd_install_ldap: false
+sssd_ldap_packages:
+  - sssd-ldap
+sssd_enable_mkhomedir: false
+sssd_mkhomedir_packages:
+  - oddjob-mkhomedir
+sssd_conf_src: "{{ appliances_environment_root }}/files/sssd.conf.j2"
+sssd_conf_dest: /etc/sssd/sssd.conf
+sssd_started: true
+sssd_enabled: true
diff --git a/ansible/roles/sssd/handlers/main.yml b/ansible/roles/sssd/handlers/main.yml
new file mode 100644
index 000000000..72c36e736
--- /dev/null
+++ b/ansible/roles/sssd/handlers/main.yml
@@ -0,0 +1,5 @@
+- name: Restart sssd
+  systemd:
+    name: sssd
+    state: restarted
+  when: sssd_started | bool
diff --git a/ansible/roles/sssd/tasks/configure.yml b/ansible/roles/sssd/tasks/configure.yml
new file mode 100644
index 000000000..ae636e9dd
--- /dev/null
+++ b/ansible/roles/sssd/tasks/configure.yml
@@ -0,0 +1,28 @@
+- name: Manage sssd.conf configuration
+  template:
+    src: "{{ sssd_conf_src }}"
+    dest: "{{ sssd_conf_dest }}"
+    owner: root
+    group: root
+    mode: u=rw,go=
+  notify: "Restart sssd"
+
+- meta: flush_handlers
+
+- name: Ensure sssd service state
+  systemd:
+    name: sssd
+    state: "{{ 'started' if sssd_started | bool else 'stopped' }}"
+    enabled: "{{ sssd_enabled | bool }}"
+
+- name: Get current authselect configuration
+  command: authselect current --raw
+  changed_when: false
+  failed_when:
+    - _authselect_current.rc != 0
+    - "'No existing configuration detected' not in _authselect_current.stdout"
+  register: _authselect_current # stdout: sssd with-mkhomedir
+
+- name: Configure nsswitch and PAM for SSSD
+  command: "authselect select sssd --force{% if sssd_enable_mkhomedir | bool %} with-mkhomedir{% endif %}"
+  when: "'sssd' not in _authselect_current.stdout"
diff --git a/ansible/roles/sssd/tasks/install.yml b/ansible/roles/sssd/tasks/install.yml
new file mode 100644
index 000000000..97aa82a2f
--- /dev/null
+++ b/ansible/roles/sssd/tasks/install.yml
@@ -0,0 +1,13 @@
+- name: Ensure sssd packages are installed
+  dnf:
+    name: "{{ sssd_packages + sssd_ldap_packages if (sssd_install_ldap | bool) else [] }}"
+
+- name: Control if sssd should start on boot
+  # Needs to be done here to prevent starting after image build, is enabled by default
+  systemd:
+    name: sssd
+    enabled: "{{ sssd_enabled | bool }}"
+
+- name: Ensure mkhomedir packages are installed if required
+  dnf:
+    name: "{{ sssd_mkhomedir_packages }}"
diff --git a/ansible/roles/sssd/tasks/main.yml b/ansible/roles/sssd/tasks/main.yml
new file mode 100644
index 000000000..2b65e84b4
--- /dev/null
+++ b/ansible/roles/sssd/tasks/main.yml
@@ -0,0 +1,2 @@
+- import_tasks: install.yml
+- import_tasks: configure.yml
diff --git a/ansible/roles/zenith_proxy/defaults/main.yml b/ansible/roles/zenith_proxy/defaults/main.yml
index dbb920c58..6b1a43aaa 100644
--- a/ansible/roles/zenith_proxy/defaults/main.yml
+++ b/ansible/roles/zenith_proxy/defaults/main.yml
@@ -15,12 +15,12 @@ zenith_proxy_pod_name: "{{ zenith_proxy_service_name }}"
 zenith_proxy_client_container_name: "{{ zenith_proxy_client_service_name }}"
 zenith_proxy_mitm_container_name: "{{ zenith_proxy_mitm_service_name }}"
 
-zenith_proxy_image_tag: '0.1.0'
+zenith_proxy_image_tag: '0.12.0'
 
-zenith_proxy_client_image_repository: ghcr.io/stackhpc/zenith-client
+zenith_proxy_client_image_repository: ghcr.io/azimuth-cloud/zenith-client
 zenith_proxy_client_image: "{{ zenith_proxy_client_image_repository }}:{{ zenith_proxy_image_tag }}"
 
-zenith_proxy_mitm_image_repository: ghcr.io/stackhpc/zenith-proxy
+zenith_proxy_mitm_image_repository: ghcr.io/azimuth-cloud/zenith-proxy
 zenith_proxy_mitm_image: "{{ zenith_proxy_mitm_image_repository }}:{{ zenith_proxy_image_tag }}"
 
 zenith_proxy_upstream_scheme: http
diff --git a/ansible/site.yml b/ansible/site.yml
index bb379399d..d973d9cb3 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -27,6 +27,7 @@
 - import_playbook: slurm.yml
 - import_playbook: portal.yml
 - import_playbook: monitoring.yml
+- import_playbook: disable-repos.yml
 
 - name: Run post.yml hook
   vars:
diff --git a/ansible/validate.yml b/ansible/validate.yml
index fae9c2f68..d02caac60 100644
--- a/ansible/validate.yml
+++ b/ansible/validate.yml
@@ -85,3 +85,11 @@
     - import_role:
         name: freeipa
         tasks_from: validate.yml
+
+- name: Validate lustre configuration
+  hosts: lustre
+  tags: lustre
+  tasks:
+    - import_role:
+        name: lustre
+        tasks_from: validate.yml
diff --git a/dev/delete-cluster.py b/dev/delete-cluster.py
index 861396efd..05f53fbfa 100755
--- a/dev/delete-cluster.py
+++ b/dev/delete-cluster.py
@@ -4,18 +4,18 @@
 Delete infrastructure for a cluster without using Terraform. Useful for CI clusters.
 
 Usage:
-    delete-cluster.py PREFIX
+    delete-cluster.py PREFIX [--force]
 
 Where PREFIX is the string at the start of the resource's names.
-It will list matching resources and prompt to confirm deletion.
+If --force is provided, it will delete all resources without confirmation.
 """
 
-import sys, json, subprocess, pprint
+import sys, json, subprocess
 
 
 CLUSTER_RESOURCES = ['server', 'port', 'volume']
 
-def delete_cluster(cluster_prefix):
+def delete_cluster(cluster_prefix, force=False):
     to_delete = {}
     for resource_type in CLUSTER_RESOURCES:
         to_delete[resource_type] = []
@@ -29,7 +29,8 @@ def delete_cluster(cluster_prefix):
             except:
                 print(resource_type, item)
                 raise
-    if input('Delete these (y/n)?:') == 'y':
+    
+    if force or input('Delete these (y/n)?:') == 'y':
         for resource_type in CLUSTER_RESOURCES:
             items = [v['ID'] for v in to_delete[resource_type]]
             if items:
@@ -40,7 +41,10 @@ def delete_cluster(cluster_prefix):
         print('Cancelled - no resources deleted')
 
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
+    if len(sys.argv) < 2 or len(sys.argv) > 3:
         print('ERROR: Incorrect argument(s).\n' + __doc__)
         exit(1)
-    delete_cluster(sys.argv[1])
+    force_flag = '--force' in sys.argv
+    cluster_prefix = sys.argv[1]
+    delete_cluster(cluster_prefix, force_flag)
+
diff --git a/dev/setup-env.sh b/dev/setup-env.sh
index bfa0758e6..6d701f2b7 100755
--- a/dev/setup-env.sh
+++ b/dev/setup-env.sh
@@ -17,6 +17,7 @@ PYTHON_VERSION=""
 if [[ "$OS" == "ubuntu" && "$MAJOR_VERSION" == "22" ]]; then
     PYTHON_VERSION="/usr/bin/python3.10"
 elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "8" ]]; then
+    # python3.9+ doesn't have selinux bindings
     PYTHON_VERSION="/usr/bin/python3.8" # use `sudo yum install python38` on Rocky Linux 8 to install this
 elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "9" ]]; then
     PYTHON_VERSION="/usr/bin/python3.9"
diff --git a/docs/experimental/compute-init.md b/docs/experimental/compute-init.md
new file mode 100644
index 000000000..c7c1d4d8c
--- /dev/null
+++ b/docs/experimental/compute-init.md
@@ -0,0 +1,18 @@
+# compute-init
+
+See the role README.md
+
+# CI workflow
+
+The compute node rebuild is tested in CI after the tests for rebuilding the
+login and control nodes. The process follows
+
+1. Compute nodes are reimaged:
+
+         ansible-playbook -v --limit compute ansible/adhoc/rebuild.yml
+
+2. Ansible-init runs against newly reimaged compute nodes
+
+3. Run sinfo and check nodes have expected slurm state
+
+         ansible-playbook -v ansible/ci/check_slurm.yml
\ No newline at end of file
diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md
new file mode 100644
index 000000000..fb2cda023
--- /dev/null
+++ b/docs/experimental/pulp.md
@@ -0,0 +1,17 @@
+# Pulp Server
+
+In order to ensure reproducible builds, the appliance can build images using repository mirrors from StackHPC's "Ark" Pulp server. The appliance can sync relevant repositories to a local Pulp server which will then be used instead of Ark.
+
+## Deploying/configuring Pulp Server
+
+### Deploying a Pulp server
+A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. This can be run with
+`ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<target_host>"`
+where `target_host` is any resolvable host. This will print a Pulp URL which can be copied to your environments as appropriate. Ensure that the server is accessible on the specified port. Note access to this server's content isn't authenticated so assumes the server is deployed behind a secure network.
+
+### Using an existing Pulp server
+An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed pulp i.e no content authentication.
+
+## Syncing Pulp content with Ark
+
+If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for Rocky 9.5 with x86_64 architecture, but can be overriden by setting extravars for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version_major`.
diff --git a/docs/image-build.md b/docs/image-build.md
new file mode 100644
index 000000000..dc968ebfd
--- /dev/null
+++ b/docs/image-build.md
@@ -0,0 +1,84 @@
+# Packer-based image build
+
+The appliance contains code and configuration to use [Packer](https://developer.hashicorp.com/packer) with the [OpenStack builder](https://www.packer.io/plugins/builders/openstack) to build images.
+
+The Packer configuration defined here builds "fat images" which contain packages, binaries and container images but no cluster-specific configuration. Using these:
+- Enables the image to be tested in CI before production use.
+- Ensures re-deployment of the cluster or deployment of additional nodes can be completed even if packages are changed in upstream repositories (e.g. due to RockyLinux or OpenHPC updates).
+- Improves deployment speed by reducing the number of package downloads to improve deployment speed.
+
+The fat images StackHPC builds and tests in CI are available from [GitHub releases](https://github.com/stackhpc/ansible-slurm-appliance/releases). However with some additional configuration it is also possible to:
+1. Build site-specific fat images from scratch.
+2. Extend an existing fat image with additional functionality.
+
+
+# Usage
+
+To build either a site-specific fat image from scratch, or to extend an existing StackHPC fat image:
+
+1. Ensure the current OpenStack credentials have sufficient authorisation to upload images (this may or may not require the `member` role for an application credential, depending on your OpenStack configuration).
+2. The provided dev credentials for StackHPC's "Ark" Pulp server must be added to the target environments. This is done by overriding `dnf_repos_username` and `dnf_repos_password` with your vault encrypted credentials in `environments/<base_environment>/inventory/group_vars/all/pulp.yml`. See the [experimental docs](experimental/pulp.md) if you wish instead wish to use a local Pulp server.
+3. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments/<environment>/builder.pkrvars.hcl` containing at a minimum:
+  
+    ```hcl
+    flavor = "general.v1.small"                           # VM flavor to use for builder VMs
+    networks = ["26023e3d-bc8e-459c-8def-dbd47ab01756"]   # List of network UUIDs to attach the VM to
+    source_image_name = "Rocky-9-GenericCloud-Base-9.4"   # Name of image to create VM with, i.e. starting image
+    inventory_groups = "control,login,compute"            # Additional inventory groups to add build VM to
+
+    ```
+
+    Note that:
+    - The network used for the Packer VM must provide outbound internet access but does not need to provide access to resources which the final cluster nodes require (e.g. Slurm control node, network filesystem servers etc.).
+    - The flavor used must have sufficent memory for the build tasks, but otherwise does not need to match the final cluster nodes. Usually 8GB is sufficent. By default, the build VM is volume-backed to allow control of the root disk size (and hence final image size) so the flavor disk size does not matter.
+    - The source image should be either a RockyLinux GenericCloud image for a site-specific image build from scratch, or a StackHPC fat image if extending an existing image.
+    - The `inventory_groups` variable takes a comma-separated list of Ansible inventory groups to add the build VM to. This is in addition to the `builder` group which it is always added to. This controls which Ansible roles and functionality run during build, and hence what gets added to the image. All possible groups are listed in `environments/common/groups` but common options for this variable will be:
+      - `update,control,login,compute`: The resultant image has all packages in the source image updated, and then packages for all types of nodes in the cluster are added. When using a GenericCloud image for `source_image_name` this builds a site-specific fat image from scratch.
+      - One or more specific groups which are not enabled in the appliance by default, e.g. `lustre`. When using a StackHPC fat image for `source_image_name` this extends the image with just this additional functionality.
+
+4. Activate the venv and the relevant environment.
+
+5. Build images using the relevant variable definition file, e.g.:
+
+        cd packer/
+        PACKER_LOG=1 /usr/bin/packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+
+    **NB:** If the build fails while creating the volume, check if the source image has the `signature_verified` property:
+
+        openstack image show $SOURCE_IMAGE
+
+      If it does, remove this property:
+
+          openstack image unset --property signature_verified $SOURCE_IMAGE
+
+      then delete the failed volume, select cancelling the build when Packer queries, and then retry. This is [OpenStack bug 1823445](https://bugs.launchpad.net/cinder/+bug/1823445).
+
+6. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc` and including a timestamp and a shortened git hash.
+
+# Build Process
+
+In summary, Packer creates an OpenStack VM, runs Ansible on that, shuts it down, then creates an image from the root disk.
+
+Many of the Packer variables defined in `openstack.pkr.hcl` control the definition of the build VM and how to SSH to it to run Ansible. These are generic OpenStack builder options
+and are not specific to the Slurm Appliance. Packer varibles can be set in a file at any convenient path; the build example above
+shows the use of the environment variable `$PKR_VAR_environment_root` (which itself sets the Packer variable
+`environment_root`) to automatically select a variable file from the current environment, but for site-specific builds
+using a path in a "parent" environment is likely to be more appropriate (as builds should not be environment-specific to allow testing before deployment to a production environment).
+
+What is Slurm Appliance-specific are the details of how Ansible is run:
+- The build VM is always added to the `builder` inventory group, which differentiates it from nodes in a cluster. This allows
+  Ansible variables to be set differently during Packer builds, e.g. to prevent services starting. The defaults for this are in `environments/common/inventory/group_vars/builder/`, which could be extended or overriden for site-specific fat image builds using `builder` groupvars for the relevant environment. It also runs some builder-specific code (e.g. to clean up the image).
+- The default fat image builds also add the build VM to the "top-level" `compute`, `control` and `login` groups. This ensures
+  the Ansible specific to all of these types of nodes run. Note other inventory groups are constructed from these by `environments/common/inventory/groups file` - this is not builder-specific.
+- As noted above, for "extra" builds the additional groups can be specified directly. In this way an existing image can be extended with site-specific Ansible, without modifying the
+  part of the image which has already been tested in the StackHPC CI.
+- The playbook `ansible/fatimage.yml` is run which is only a subset of `ansible/site.yml`. This allows restricting the code which runs during build for cases where setting `builder`
+  groupvars is not sufficient (e.g. a role always attempts to configure or start services).
+
+There are some things to be aware of when developing Ansible to run in a Packer build VM:
+  - Only some tasks make sense. E.g. any services with a reliance on the network cannot be started, and should not be enabled if, when creating an instance with the resulting image, the remote service will not be immediately present.
+  - Nothing should be written to the persistent state directory `appliances_state_dir`, as this is on the root filesystem rather than an OpenStack volume.
+  - Care should be taken not to leave data on the root filesystem which is not wanted in the final image (e.g secrets).
+  - Build VM hostnames are not the same as for equivalent "real" hosts and do not contain `login`, `control` etc. Therefore variables used by the build VM must be defined as groupvars not hostvars.
+  - Ansible may need to use a proxyjump to reach cluster nodes, which can be defined via Ansible's `ansible_ssh_common_args` variable. If Packer should not use the same proxy
+    to connect to build VMs (e.g. because build happens on a different network), this proxy configuration should not be added to the `all` group.
diff --git a/docs/k3s.README.md b/docs/k3s.README.md
new file mode 100644
index 000000000..1b6651159
--- /dev/null
+++ b/docs/k3s.README.md
@@ -0,0 +1,8 @@
+# Overview
+A K3s cluster is deployed with the Slurm cluster. Both an agent and server instance of K3s is installed during image build and the correct service (determined by OpenStack metadata) will be 
+enabled during boot. Nodes with the `k3s_server` metadata field defined will be configured as K3s agents (this field gives them the address of the server). The Slurm control node is currently configured as a server while all other nodes are configured as agents. Using multiple K3s servers isn't supported. Currently only the root user on the control node has 
+access to the Kubernetes API. The `k3s` role installs Helm for package management. K9s is also installed in the image and can be used by the root user.
+
+# Idempotency
+K3s is intended to only be installed during image build as it is configured by the appliance on first boot with `azimuth_cloud.image_utils.linux_ansible_init`. Therefore, the `k3s` role isn't
+idempotent and changes to variables will not be reflected in the image when running `site.yml`.
diff --git a/docs/monitoring-and-logging.md b/docs/monitoring-and-logging.md
index 3e3de38c0..db228d410 100644
--- a/docs/monitoring-and-logging.md
+++ b/docs/monitoring-and-logging.md
@@ -96,7 +96,7 @@ The `grafana` group controls the placement of the grafana service. Load balancin
 
 ### Access
 
-If Open Ondemand is enabled then by default this is used to proxy Grafana, otherwise Grafana is accessed through the first . See `grafana_url` in [environments/common/inventory/group_vars/all/grafana.yml](../environments/common/inventory/group_vars/all/grafana.yml). The port used (variable `grafana_port`) defaults to `3000`.
+If Open OnDemand is enabled then by default this is used to proxy Grafana, otherwise Grafana is accessed through the first . See `grafana_url` in [environments/common/inventory/group_vars/all/grafana.yml](../environments/common/inventory/group_vars/all/grafana.yml). The port used (variable `grafana_port`) defaults to `3000`.
 
 The default credentials for the admin user are:
 
diff --git a/docs/openondemand.README.md b/docs/openondemand.md
similarity index 61%
rename from docs/openondemand.README.md
rename to docs/openondemand.md
index 5daba3408..6b501d20b 100644
--- a/docs/openondemand.README.md
+++ b/docs/openondemand.md
@@ -1,28 +1,28 @@
 # Overview
 
-The appliance can deploy the Open Ondemand portal. This page describes how to enable this and the default appliance configuration/behaviour. Note that detailed configuration documentation is provided by:
+The appliance can deploy the Open OnDemand portal. This page describes how to enable this and the default appliance configuration/behaviour. Note that detailed configuration documentation is provided by:
 
 - The README for the included `openondemand` role in this repo - [ansible/roles/openondemand/README.md](../ansible/roles/openondemand/README.md).
 - The README and default variables for the underlying "official" role which the above wraps - [Open OnDemand Ansible Role](https://github.com/OSC/ood-ansible)
-- The documentation for Open Ondemand [itself](https://osc.github.io/ood-documentation/latest/index.html)
+- The documentation for Open OnDemand [itself](https://osc.github.io/ood-documentation/latest/index.html)
 
 This appliance can deploy and configure:
-- The Open Ondemand server itself (usually on a single login node).
+- The Open OnDemand server itself (usually on a single login node).
 - User authentication using one of:
     - An external OIDC provider.
     - HTTP basic authenication and PAM.
 - Virtual desktops on compute nodes.
 - Jupyter nodebook servers on compute nodes.
-- Proxying of Grafana (usually deployed on the control node) via the Open Ondemand portal.
-- Links to additional filesystems and pages from the Open Ondemand Dashboard.
-- A Prometheus exporter for the Open Ondemand server and related Grafana dashboard
+- Proxying of Grafana (usually deployed on the control node) via the Open OnDemand portal.
+- Links to additional filesystems and pages from the Open OnDemand Dashboard.
+- A Prometheus exporter for the Open OnDemand server and related Grafana dashboard
 
 For examples of all of the above see the `smslabs-example` environment in this repo.
 
-# Enabling Open Ondemand
-To enable the Open Ondemand server, add single host to the `openondemand` inventory group. Generally, this should be a node in the `login` group, as Open Ondemand must be able to access Slurm commands.
+# Enabling Open OnDemand
+To enable the Open OnDemand server, add single host to the `openondemand` inventory group. Generally, this should be a node in the `login` group, as Open OnDemand must be able to access Slurm commands.
 
-To enable compute nodes for virtual desktops or Jupyter notebook servers (accessed through the Open Ondemand portal), add nodes/groups to the `openondemand_desktop` and `openondemand_jupyter` inventory groups respectively. These may be all or a subset of the `compute` group.
+To enable compute nodes for virtual desktops or Jupyter notebook servers (accessed through the Open OnDemand portal), add nodes/groups to the `openondemand_desktop` and `openondemand_jupyter` inventory groups respectively. These may be all or a subset of the `compute` group.
 
 The above functionality is configured by running the `ansible/portal.yml` playbook. This is automatically run as part of `ansible/site.yml`.
 
@@ -30,11 +30,10 @@ The above functionality is configured by running the `ansible/portal.yml` playbo
 
 See the [ansible/roles/openondemand/README.md](../ansible/roles/openondemand/README.md) for more details on the variables described below.
 
-At minimum the following must be defined:
-- `openondemand_servername` - this must be defined for both `openondemand` and `grafana` hosts (when Grafana is enabled). It is suggested to place it groupvars for `all`.
-- `openondemand_auth` and any corresponding options.
-- `openondemand_desktop_partition` and `openondemand_jupyter_partition` if the corresponding inventory groups are defined.
-- `openondemand_host_regex` if `openondemand_desktop` or `openondemand_jupyter` inventory groups are defined and/or proxying Grafana via Open Ondemand is required.
+The following variables have been given default values to allow Open OnDemand to work in a newly created environment without additional configuration, but generally should be overridden in `environment/site/inventory/group_vars/all/` with site-specific values:
+- `openondemand_servername` - this must be defined for both `openondemand` and `grafana` hosts (when Grafana is enabled). Default is `ansible_host` (i.e. the IP address) of the first host in the `openondemand` group.
+- `openondemand_auth` and any corresponding options. Defaults to `basic_pam`.
+- `openondemand_desktop_partition` and `openondemand_jupyter_partition` if the corresponding inventory groups are defined. Defaults to the first compute group defined in the `compute` Terraform variable in `environments/$ENV/terraform`.
 
 It is also recommended to set:
 - `openondemand_dashboard_support_url`
@@ -42,6 +41,9 @@ It is also recommended to set:
 
 If shared filesystems other than `$HOME` are available, add paths to `openondemand_filesapp_paths`.
 
-The appliance automatically configures Open Ondemand to proxy Grafana and adds a link to it on the Open Ondemand dashboard. This means no external IP (or SSH proxying etc) is required to access Grafana (which by default is deployed on the control node). To allow users to authenticate to Grafana, the simplest option is to enable anonymous (View-only) login by setting `grafana_auth_anonymous` (see [environments/common/inventory/group_vars/all/grafana.yml](../environments/common/inventory/group_vars/all/grafana.yml)[^1]).
+The appliance automatically configures Open OnDemand to proxy Grafana and adds a link to it on the Open OnDemand dashboard. This means no external IP (or SSH proxying etc) is required to access Grafana (which by default is deployed on the control node). To allow users to authenticate to Grafana, the simplest option is to enable anonymous (View-only) login by setting `grafana_auth_anonymous` (see [environments/common/inventory/group_vars/all/grafana.yml](../environments/common/inventory/group_vars/all/grafana.yml)[^1]).
 
-[^1]: Note that if `openondemand_auth` is `basic_pam` and anonymous Grafana login is enabled, the appliance will (by default) configure Open Ondemand's Apache server to remove the Authorisation header from proxying of all `node/` addresses. This is done as otherwise Grafana tries to use this header to authenticate, which fails with the default configuration where only the admin Grafana user `grafana` is created. Note that the removal of this header in this configuration means it cannot be used to authenticate proxied interactive applications - however the appliance-deployed remote desktop and Jupyter Notebook server applications use other authentication methods. An alternative if using `basic_pam` is not to enable anonymous Grafana login and to create Grafana users matching the local users (e.g. in `environments/<env>/hooks/post.yml`).
+[^1]: Note that if `openondemand_auth` is `basic_pam` and anonymous Grafana login is enabled, the appliance will (by default) configure Open OnDemand's Apache server to remove the Authorisation header from proxying of all `node/` addresses. This is done as otherwise Grafana tries to use this header to authenticate, which fails with the default configuration where only the admin Grafana user `grafana` is created. Note that the removal of this header in this configuration means it cannot be used to authenticate proxied interactive applications - however the appliance-deployed remote desktop and Jupyter Notebook server applications use other authentication methods. An alternative if using `basic_pam` is not to enable anonymous Grafana login and to create Grafana users matching the local users (e.g. in `environments/<env>/hooks/post.yml`).
+
+# Access
+By default the appliance authenticates against OOD with basic auth through PAM. When creating a new environment, a new user with username `demo_user` will be created. Its password is found under `vault_openondemand_default_user` in the appliance secrets store in `environments/{ENV}/inventory/group_vars/all/secrets.yml`. Other users can be defined by overriding the `basic_users_users` variable in your environment (templated into `environments/{ENV}/inventory/group_vars/all/basic_users.yml` by default).
diff --git a/docs/operations.md b/docs/operations.md
index a20d7f10c..7a0a5b919 100644
--- a/docs/operations.md
+++ b/docs/operations.md
@@ -63,17 +63,46 @@ This is a usually a two-step process:
 Deploying the additional nodes and applying these changes requires rerunning both Terraform and the Ansible site.yml playbook - follow [Deploying a Cluster](#Deploying-a-Cluster).
 
 # Adding Additional Packages
-Packages from any enabled DNF repositories (which always includes EPEL, PowerTools and OpenHPC) can be added to all nodes by defining a list `openhpc_packages_extra` (defaulted to the empty list in the common environment) in e.g. `environments/$SITE_ENV/inventory/group_vars/all/openhpc.yml`. For example:
-
-    # environments/foo-base/inventory/group_vars/all/openhpc.yml:
-    openhpc_packages_extra:
-    - somepackage
-    - anotherpackage
-
-
-The packages available from the OpenHPC repos are described in Appendix E of the OpenHPC installation guide (linked from the [OpenHPC releases page](https://github.com/openhpc/ohpc/releases/)). Note "user-facing" OpenHPC packages such as compilers, mpi libraries etc. include corresponding `lmod` modules.
-
-To add these packages to the current cluster, run the same command as for [Reconfiguring Slurm](#Reconfiguring-Slurm). TODO: describe what's required to add these to site-specific images.
+By default, the following utility packages are installed during the StackHPC image build:
+- htop
+- nano
+- screen
+- tmux
+- wget
+- bind-utils
+- net-tools
+- postfix
+- git
+- latest python version for system (3.6 for for Rocky 8.9 and 3.12 for Rocky 9.4)
+
+Additional packages can be added during image builds by:
+- adding the `extra_packages` group to the build `inventory_groups` (see
+[docs/image-build.md](./image-build.md))
+- defining a list of packages in `appliances_extra_packages_other` in e.g.
+`environments/$SITE_ENV/inventory/group_vars/all/defaults.yml`. For example:
+
+    ```yaml
+        # environments/foo-base/inventory/group_vars/all/defaults.yml:
+        appliances_extra_packages_other:
+        - somepackage
+        - anotherpackage
+    ```
+
+For packages which come from repositories mirroed by StackHPC's "Ark" Pulp server
+(including rocky, EPEL and OpenHPC repositories), this will require either [Ark
+credentials](./image-build.md)) or a [local Pulp mirror](./experimental/pulp.md)
+to be configured. This includes rocky, EPEL and OpenHPC repos.
+
+The packages available from the OpenHPC repos are described in Appendix E of
+the OpenHPC installation guide (linked from the
+[OpenHPC releases page](https://github.com/openhpc/ohpc/releases/)). Note
+"user-facing" OpenHPC packages such as compilers, mpi libraries etc. include
+corresponding `lmod` modules.
+
+Packages *may* also be installed during the site.yml, by adding the `cluster`
+group into the `extra_packages` group. An error will occur if Ark credentials
+are defined in this case, as they are readable by unprivileged users in the
+`.repo` files and a local Pulp mirror must be used instead.
 
 If additional repositories are required, these could be added/enabled as necessary in a play added to `environments/$SITE_ENV/hooks/{pre,post}.yml` as appropriate. Note such a plat should NOT exclude the builder group, so that the repositories are also added to built images. There are various Ansible modules which might be useful for this:
     - `ansible.builtin.yum_repository`: Add a repo from an URL providing a 'repodata' directory.
diff --git a/docs/production.md b/docs/production.md
index 7219ee7fc..59b9f3775 100644
--- a/docs/production.md
+++ b/docs/production.md
@@ -1,9 +1,129 @@
 # Production Deployments
 
-This page contains some brief notes about differences between the default/demo configuration, as described in the main [README.md](../README.md) and production-ready deployments.
+This page contains some brief notes about differences between the default/demo
+configuration (as described in the main [README.md](../README.md)) and
+production-ready deployments.
 
-- Create a site environment. Usually at least production, staging and possibly development environments are required. To avoid divergence of configuration these should all have an `inventory` path referencing a shared, site-specific base environment. Where possible hooks should also be placed in this site-specific environment.
-- Vault-encrypt secrets. Running the `generate-passwords.yml` playbook creates a secrets file at `environments/$ENV/inventory/group_vars/all/secrets.yml`. To ensure staging environments are a good model for production this should generally be moved into the site-specific environment. It can be be encrypted using [Ansible vault](https://docs.ansible.com/ansible/latest/user_guide/vault.html) and then committed to the repository.
-- Ensure created instances have accurate/synchronised time. For VM instances this is usually provided by the hypervisor, but if not (or for bare metal instances) it may be necessary to configure or proxy `chronyd` via an environment hook.
-- Remove production volumes from OpenTofu control. In the default OpenTofu configuration, deleting the resources also deletes the volumes used for persistent state and home directories. This is usually undesirable for production, so these resources should be removed from the OpenTofu configurations and manually deployed once. However note that for development environments leaving them under OpenTofu control is usually best.
-- Configure Open OpenOndemand - see [specific documentation](openondemand.README.md).
+- Get it agreed up front what the cluster names will be. Changing this later
+  requires instance deletion/recreation.
+
+- At least three environments should be created:
+    - `site`: site-specific base environment
+    - `production`: production environment
+    - `staging`: staging environment
+
+  A `dev` environment should also be created if considered required, or this
+  can be left until later.,
+
+  These can all be produced using the cookicutter instructions, but the
+  `production` and `staging` environments will need their
+  `environments/$ENV/ansible.cfg` file modifying so that they point to the
+  `site` environment:
+
+    ```ini
+    inventory = ../common/inventory,../site/inventory,inventory
+    ```
+
+- To avoid divergence of configuration all possible overrides for group/role
+vars should be placed in `environments/site/inventory/group_vars/all/*.yml`
+unless the value really is environment-specific (e.g. DNS names for
+`openondemand_servername`).
+
+- Where possible hooks should also be placed in `environments/site/hooks/`
+and referenced from the `site` and `production` environments, e.g.:
+
+    ```yaml
+    # environments/production/hooks/pre.yml:
+    - name: Import parent hook
+      import_playbook: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/../site/hooks/pre.yml"
+    ```
+
+- OpenTofu configurations should be defined in the `site` environment and used
+  as a module from the other environments. This can be done with the
+  cookie-cutter generated configurations:
+  - Delete the *contents* of the cookie-cutter generated `terraform/` directories
+    from the `production` and `staging` environments.
+  - Create a `main.tf` in those directories which uses `site/terraform/` as a
+    [module](https://opentofu.org/docs/language/modules/), e.g. :
+
+    ```
+    ...
+    module "cluster" {
+        source = "../../site/terraform/"
+
+        cluster_name = "foo"
+        ...
+    }
+    ```
+
+    Note that:
+        - Environment-specific variables (`cluster_name`) should be hardcoded
+          into the module block.
+        - Environment-independent variables (e.g. maybe `cluster_net` if the
+          same is used for staging and production) should be set as *defaults*
+          in `environments/site/terraform/variables.tf`, and then don't need to
+          be passed in to the module.
+
+- Vault-encrypt secrets. Running the `generate-passwords.yml` playbook creates
+  a secrets file at `environments/$ENV/inventory/group_vars/all/secrets.yml`.
+  To ensure staging environments are a good model for production this should
+  generally be moved into the `site` environment. It should be be encrypted
+  using [Ansible vault](https://docs.ansible.com/ansible/latest/user_guide/vault.html)
+  and then committed to the repository.
+
+- Ensure created instances have accurate/synchronised time. For VM instances
+  this is usually provided by the hypervisor, but if not (or for bare metal
+  instances) it may be necessary to configure or proxy `chronyd` via an
+  environment hook.
+
+- The cookiecutter provided OpenTofu configurations define resources for home and
+  state volumes. The former may not be required if the cluster's `/home` is
+  provided from an external filesystem (or Manila). In any case, in at least
+  the production environment, and probably also in the staging environment,
+  the volumes should be manually created and the resources changed to [data
+  resources](https://opentofu.org/docs/language/data-sources/). This ensures that even if the cluster is deleted via tofu, the
+  volumes will persist.
+
+  For a development environment, having volumes under tofu control via volume
+  resources is usually appropriate as there may be many instantiations
+  of this environment.
+
+- Enable `etc_hosts` templating:
+
+    ```yaml
+    # environments/site/inventory/groups:
+    [etc_hosts:children]
+    cluster
+    ```
+
+- Configure Open OnDemand - see [specific documentation](openondemand.README.md).
+
+- Remove the `demo_user` user from `environments/$ENV/inventory/group_vars/all/basic_users.yml`
+
+- Consider whether having (read-only) access to Grafana without login is OK. If not, remove `grafana_auth_anonymous` in `environments/$ENV/inventory/group_vars/all/grafana.yml`
+
+- Modify `environments/site/terraform/nodes.tf` to provide fixed IPs for at least
+  the control node, and (if not using FIPs) the login node(s):
+
+    ```
+    resource "openstack_networking_port_v2" "control" {
+        ...
+        fixed_ip {
+            subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id
+            ip_address = var.control_ip_address
+        }
+    }
+    ```
+    
+  Note the variable `control_ip_address` is new.
+
+  Using fixed IPs will require either using admin credentials or policy changes.
+
+- If floating IPs are required for login nodes, modify the OpenTofu configurations
+  appropriately.
+
+- Consider whether mapping of baremetal nodes to ironic nodes is required. See
+  [PR 485](https://github.com/stackhpc/ansible-slurm-appliance/pull/485).
+
+- Note [PR 473](https://github.com/stackhpc/ansible-slurm-appliance/pull/473)
+  may help identify any site-specific configuration. 
diff --git a/docs/upgrades.md b/docs/upgrades.md
new file mode 100644
index 000000000..6e398934e
--- /dev/null
+++ b/docs/upgrades.md
@@ -0,0 +1,103 @@
+# Upgrades
+
+This document explains the generic steps required to upgrade a deployment of the Slurm Appliance with upstream changes from StackHPC.
+Generally, upstream releases will happen roughly monthly. Releases may contain new functionality and/or updated images.
+
+Any site-specific instructions in [docs/site/README.md](site/README.md) should be reviewed in tandem with this.
+
+This document assumes the deployment repository has:
+1. Remotes:
+    - `origin` referring to the site-specific remote repository.
+    - `stackhpc` referring to the StackHPC repository at https://github.com/stackhpc/ansible-slurm-appliance.git.
+2. Branches:
+    - `main` - following `main/origin`, the current site-specific code deployed to production.
+    - `upstream` - following `main/stackhpc`, i.e. the upstream `main` branch from `stackhpc`.
+3. The following environments:
+    - `$PRODUCTION`: a production environment, as defined by e.g. `environments/production/`.
+    - `$STAGING`: a production environment, as defined by e.g. `environments/staging/`.
+    - `$SITE_ENV`: a base site-specific environment, as defined by e.g. `environments/mysite/`.
+
+**NB:** Commands which should be run on the Slurm login node are shown below prefixed `[LOGIN]$`.
+All other commands should be run on the Ansible deploy host.
+
+1. Update the `upstream` branch from the `stackhpc` remote, including tags:
+
+        git fetch stackhpc main --tags
+
+1. Identify the latest release from the [Slurm appliance release page](https://github.com/stackhpc/ansible-slurm-appliance/releases). Below this release is shown as `vX.Y`.
+
+1. Ensure your local site branch is up to date and create a new branch from it for the
+   site-specfic release code:
+
+        git checkout main
+        git pull --prune
+        git checkout -b update/vX.Y
+
+1. Merge the upstream code into your release branch:
+
+        git merge vX.Y
+
+   It is possible this will introduce merge conflicts; fix these following the usual git 
+   prompts. Generally merge conflicts should only exist where functionality which was added
+   for your site (not in a hook) has subsequently been merged upstream.
+
+1. Push this branch and create a PR:
+
+        git push
+        # follow instructions
+
+1. Review the PR to see if any added/changed functionality requires alteration of
+   site-specific configuration. In general changes to existing functionality will aim to be
+   backward compatible. Alteration of site-specific configuration will usually only be
+   necessary to use new functionality or where functionality has been upstreamed as above.
+
+   Make changes as necessary.
+
+1. Identify image(s) from the relevant [Slurm appliance release](https://github.com/stackhpc/ansible-slurm-appliance/releases), and download
+   using the link on the release plus the image name, e.g. for an image `openhpc-ofed-RL8-240906-1042-32568dbb`:
+
+        wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/openhpc-images/openhpc-ofed-RL8-240906-1042-32568dbb
+
+    Note that some releases may not include new images. In this case use the image from the latest previous release with new images.
+
+1. If required, build an "extra" image with local modifications, see [docs/image-build.md](./image-build.md).
+
+1. Modify your site-specific environment to use this image, e.g. via `cluster_image_id` in `environments/$SITE_ENV/terraform/variables.tf`.
+
+1. Test this in your staging cluster.
+
+1. Commit changes and push to the PR created above.
+
+1. Declare a future outage window to cluster users. A [Slurm reservation](https://slurm.schedmd.com/scontrol.html#lbAQ) can be
+   used to prevent jobs running during that window, e.g.:
+
+        [LOGIN]$  sudo scontrol create reservation Flags=MAINT ReservationName="upgrade-vX.Y" StartTime=2024-10-16T08:00:00 EndTime=2024-10-16T10:00:00 Nodes=ALL Users=root
+
+   Note a reservation cannot be created if it may overlap with currently running jobs (defined by job or partition time limits).
+
+1. At the outage window, check there are no jobs running:
+
+        [LOGIN]$ squeue
+
+1. Deploy the branch created above to production, i.e. activate the production environment, run OpenTofu to reimage or
+delete/recreate instances with the new images (depending on how the root disk is defined), and run Ansible's `site.yml`
+playbook to reconfigure the cluster, e.g. as described in the main [README.md](../README.md).
+
+1. Check slurm is up:
+
+        [LOGIN]$ sinfo -R
+   
+   The `-R` shows the reason for any nodes being down.
+
+1. If the above shows nodes done for having been "unexpectedly rebooted", set them up again:
+
+        [LOGIN]$ sudo scontrol update state=RESUME nodename=$HOSTLIST_EXPR
+
+    where the hostlist expression might look like e.g. `general-[0-1]` to reset state for nodes 0 and 1 of the general partition.
+
+1. Delete the reservation:
+
+        [LOGIN]$ sudo scontrol delete ReservationName="upgrade-slurm-v1.160"
+
+1. Tell users the cluster is available again.
+
diff --git a/environments/.caas/hooks/pre.yml b/environments/.caas/hooks/pre.yml
index 05b0255c8..c8951a4af 100644
--- a/environments/.caas/hooks/pre.yml
+++ b/environments/.caas/hooks/pre.yml
@@ -1,5 +1,13 @@
 ---
 
+# Generate k3s token
+- name: Generate k3s token
+  # NB: Although this generates a new token on each run, the actual token set in metadata is retrieved from a set-once tofu resource, hence only the first value ever generated is relevant.
+  hosts: openstack
+  tasks:
+    - ansible.builtin.set_fact:
+        k3s_token: "{{ lookup('ansible.builtin.password', '/dev/null', length=64) }}"
+
 # Provision the infrastructure using Terraform
 - name: Provision infrastructure
   hosts: openstack
diff --git a/environments/.caas/inventory/extra_groups b/environments/.caas/inventory/extra_groups
index d60ae7839..45a1dc7aa 100644
--- a/environments/.caas/inventory/extra_groups
+++ b/environments/.caas/inventory/extra_groups
@@ -14,6 +14,3 @@ compute
 
 [podman:children]
 zenith
-
-[persist_hostkeys:children]
-openondemand
diff --git a/environments/.caas/inventory/group_vars/all/nfs.yml b/environments/.caas/inventory/group_vars/all/nfs.yml
index 14fff6295..f42422601 100644
--- a/environments/.caas/inventory/group_vars/all/nfs.yml
+++ b/environments/.caas/inventory/group_vars/all/nfs.yml
@@ -1,14 +1,5 @@
 nfs_server: "{{ nfs_server_default }}"
 
-caas_nfs_ood_state:
-  - comment: Export /var/lib/state from Slurm control node to OOD
-    nfs_enable:
-        server:  "{{ inventory_hostname in groups['control'] }}"
-        clients: "{{ inventory_hostname in groups['openondemand'] }}"
-    nfs_export: "{{ appliances_state_dir }}"
-    nfs_client_mnt_point: "{{ appliances_state_dir }}"
-    nfs_client_mnt_options: "x-systemd.required-by=zenith-ood.service,x-systemd.before=zenith-ood.service"
-
 caas_nfs_home:
   - comment: Export /exports/home from Slurm control node as /home
     nfs_enable:
@@ -17,4 +8,4 @@ caas_nfs_home:
     nfs_export: "/exports/home" # assumes skeleton TF is being used
     nfs_client_mnt_point: "/home"
 
-nfs_configurations: "{{ caas_nfs_ood_state + (caas_nfs_home if not cluster_home_manila_share | bool else []) }}"
+nfs_configurations: "{{ caas_nfs_home if not cluster_home_manila_share | bool else [] }}"
diff --git a/environments/.caas/inventory/group_vars/all/selinux.yml b/environments/.caas/inventory/group_vars/all/selinux.yml
deleted file mode 100644
index 1f1098126..000000000
--- a/environments/.caas/inventory/group_vars/all/selinux.yml
+++ /dev/null
@@ -1 +0,0 @@
-selinux_state: disabled
\ No newline at end of file
diff --git a/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl b/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl
index 5adf4199c..db0b28b49 100644
--- a/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl
+++ b/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl
@@ -4,4 +4,7 @@ networks = ["909e49e8-6911-473a-bf88-0495ca63853c"] # slurmapp-ci
 ssh_keypair_name = "slurm-app-ci"
 ssh_private_key_file = "~/.ssh/id_rsa"
 security_groups = ["default", "SSH"]
-floating_ip_network = "external"
+# see environments/.stackhpc/inventory/group_vars/all/bastion.yml:
+ssh_bastion_username = "slurm-app-ci"
+ssh_bastion_host = "195.114.30.222"
+ssh_bastion_private_key_file = "~/.ssh/id_rsa"
diff --git a/environments/.stackhpc/SMS.pkrvars.hcl b/environments/.stackhpc/SMS.pkrvars.hcl
index b88106fe8..3ebe734eb 100644
--- a/environments/.stackhpc/SMS.pkrvars.hcl
+++ b/environments/.stackhpc/SMS.pkrvars.hcl
@@ -2,6 +2,7 @@ flavor = "general.v1.small"
 networks = ["e2b9e59f-43da-4e1c-b558-dc9da4c0d738"] # stackhpc-ipv4-geneve
 ssh_keypair_name = "slurm-app-ci"
 ssh_private_key_file = "~/.ssh/id_rsa"
+# see environments/.stackhpc/inventory/group_vars/all/bastion.yml:
 ssh_bastion_username = "slurm-app-ci"
 ssh_bastion_host = "185.45.78.150"
-ssh_bastion_private_key_file = "~/.ssh/id_rsa"
\ No newline at end of file
+ssh_bastion_private_key_file = "~/.ssh/id_rsa"
diff --git a/environments/.stackhpc/hooks/post.yml b/environments/.stackhpc/hooks/post.yml
index 5032f0ecc..9d506d725 100644
--- a/environments/.stackhpc/hooks/post.yml
+++ b/environments/.stackhpc/hooks/post.yml
@@ -9,8 +9,6 @@
         path: "{{ item }}"
         state: absent
       with_items:
-        - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.7-1/gems/bootstrap_form-2.7.0/test/dummy/Gemfile.lock
-        - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.9-1/gems/bootstrap_form-2.7.0/test/dummy/Gemfile.lock
-        - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.7-1/gems/bootstrap_form-4.5.0/demo/yarn.lock
-        - /opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/3.1.9-1/gems/bootstrap_form-4.5.0/demo/yarn.lock
-        - /var/www/ood/apps/sys/dashboard/node_modules/data-confirm-modal/Gemfile.lock
\ No newline at end of file
+        - "/opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/{{ ondemand_package_version }}-1/gems/bootstrap_form-2.7.0/test/dummy/Gemfile.lock"
+        - "/opt/ood/ondemand/root/usr/share/gems/3.1/ondemand/{{ ondemand_package_version }}-1/gems/bootstrap_form-4.5.0/demo/yarn.lock"
+        - /var/www/ood/apps/sys/dashboard/node_modules/data-confirm-modal/Gemfile.lock
diff --git a/environments/.stackhpc/inventory/extra_groups b/environments/.stackhpc/inventory/extra_groups
index 7c9a7c774..2531b803e 100644
--- a/environments/.stackhpc/inventory/extra_groups
+++ b/environments/.stackhpc/inventory/extra_groups
@@ -31,3 +31,7 @@ compute
 [squid:children]
 # Install squid into fat image
 builder
+
+[sssd:children]
+# Install sssd into fat image
+builder
diff --git a/environments/.stackhpc/inventory/group_vars/all/basic_users.yml b/environments/.stackhpc/inventory/group_vars/all/basic_users.yml
index ae416cf72..e2088ffd9 100644
--- a/environments/.stackhpc/inventory/group_vars/all/basic_users.yml
+++ b/environments/.stackhpc/inventory/group_vars/all/basic_users.yml
@@ -1,6 +1,6 @@
-test_user_password: "{{ lookup('env', 'TESTUSER_PASSWORD') | default(vault_testuser_password, true) }}" # CI uses env, debug can set vault_testuser_password
+test_demo_user_password: "{{ lookup('env', 'DEMO_USER_PASSWORD') | default(vault_demo_user_password, true) }}" # CI uses env, debug can set vault_demo_user_password
 
 basic_users_users:
-  - name: testuser # can't use rocky as $HOME isn't shared!
-    password: "{{ test_user_password | password_hash('sha512', 65534 | random(seed=inventory_hostname) | string) }}" # idempotent
+  - name: demo_user # can't use rocky as $HOME isn't shared!
+    password: "{{ test_demo_user_password | password_hash('sha512', 65534 | random(seed=inventory_hostname) | string) }}" # idempotent
     uid: 1005
diff --git a/environments/.stackhpc/inventory/group_vars/all/freeipa.yml b/environments/.stackhpc/inventory/group_vars/all/freeipa.yml
index 4b3750650..9a979ab16 100644
--- a/environments/.stackhpc/inventory/group_vars/all/freeipa.yml
+++ b/environments/.stackhpc/inventory/group_vars/all/freeipa.yml
@@ -2,8 +2,8 @@
 
 # NB: Users defined this way have expired passwords
 freeipa_users:
-  - name: testuser # can't use rocky as $HOME isn't shared!
-    password: "{{ test_user_password }}"
+  - name: demo_user # can't use rocky as $HOME isn't shared!
+    password: "{{ test_demo_user_password }}"
     givenname: test
     sn: test
 
diff --git a/environments/.stackhpc/inventory/group_vars/grafana/overrides.yml b/environments/.stackhpc/inventory/group_vars/all/grafana.yml
similarity index 100%
rename from environments/.stackhpc/inventory/group_vars/grafana/overrides.yml
rename to environments/.stackhpc/inventory/group_vars/all/grafana.yml
diff --git a/environments/.stackhpc/inventory/group_vars/all/manila.yml b/environments/.stackhpc/inventory/group_vars/all/manila.yml
new file mode 100644
index 000000000..59f935873
--- /dev/null
+++ b/environments/.stackhpc/inventory/group_vars/all/manila.yml
@@ -0,0 +1,7 @@
+os_manila_mount_shares_arcus:
+  - share_name: slurm-v2-home
+    mount_path: /project
+  - share_name: slurm-scratch
+    mount_path: /scratch
+
+os_manila_mount_shares: "{{ os_manila_mount_shares_arcus if ci_cloud == 'ARCUS' else [] }}"
diff --git a/environments/.stackhpc/inventory/group_vars/openhpc/overrides.yml b/environments/.stackhpc/inventory/group_vars/all/openhpc.yml
similarity index 100%
rename from environments/.stackhpc/inventory/group_vars/openhpc/overrides.yml
rename to environments/.stackhpc/inventory/group_vars/all/openhpc.yml
diff --git a/environments/.stackhpc/inventory/group_vars/all/openondemand.yml b/environments/.stackhpc/inventory/group_vars/all/openondemand.yml
index 11d475664..72b6cf476 100644
--- a/environments/.stackhpc/inventory/group_vars/all/openondemand.yml
+++ b/environments/.stackhpc/inventory/group_vars/all/openondemand.yml
@@ -1 +1,8 @@
-openondemand_servername: "{{ hostvars[ groups['openondemand'] | first].ansible_host }}" # Use a SOCKS proxy to acccess
+openondemand_auth: basic_pam
+openondemand_jupyter_partition: standard
+openondemand_desktop_partition: standard
+#openondemand_dashboard_support_url: 
+#openondemand_dashboard_docs_url:
+#openondemand_filesapp_paths:
+ondemand_package: ondemand-"{{ ondemand_package_version }}"
+ondemand_package_version: '3.1.10'
diff --git a/environments/.stackhpc/inventory/group_vars/builder.yml b/environments/.stackhpc/inventory/group_vars/builder.yml
index 8d7ee98d2..10b15adac 100644
--- a/environments/.stackhpc/inventory/group_vars/builder.yml
+++ b/environments/.stackhpc/inventory/group_vars/builder.yml
@@ -1 +1,21 @@
 #update_enable: false # Can uncomment for speed debugging non-update related build issues
+sssd_install_ldap: true # include sssd-ldap package in fatimage
+# update_enable: false # Can uncomment for speed debugging non-update related build issues
+
+# Uncomment below to use CI pulp servers
+
+# pulp_server_config:
+#   LEAFCLOUD:
+#     url: http://192.168.10.157:8080
+#     password: lookup('env','LEAFCLOUD_PULP_PASSWORD')
+
+# appliances_pulp_url: "{{ pulp_server_config[lookup('env','CI_CLOUD')].url }}"
+# pulp_site_password: "{{ pulp_server_config[lookup('env','CI_CLOUD')].password }}"
+
+# Alternatively, configure to use ark directly:
+dnf_repos_username: slurm-app-ci
+dnf_repos_password: "{{ lookup('env','ARK_PASSWORD') }}"
+
+# Can be set regardless of approach above:
+pulp_site_upstream_username: slurm-app-ci
+pulp_site_upstream_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
diff --git a/environments/.stackhpc/inventory/group_vars/openondemand/overrides.yml b/environments/.stackhpc/inventory/group_vars/openondemand/overrides.yml
deleted file mode 100644
index 735da25df..000000000
--- a/environments/.stackhpc/inventory/group_vars/openondemand/overrides.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-openondemand_auth: basic_pam
-openondemand_jupyter_partition: standard
-openondemand_desktop_partition: standard
-#openondemand_dashboard_support_url: 
-#openondemand_dashboard_docs_url:
-#openondemand_filesapp_paths:
diff --git a/environments/.stackhpc/inventory/group_vars/selinux/overrides.yml b/environments/.stackhpc/inventory/group_vars/selinux/overrides.yml
deleted file mode 100644
index c3b28b913..000000000
--- a/environments/.stackhpc/inventory/group_vars/selinux/overrides.yml
+++ /dev/null
@@ -1 +0,0 @@
-selinux_state: disabled
diff --git a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
index c07b2c4ac..37bd8c3d6 100644
--- a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
+++ b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
@@ -1,7 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-241022-0441-a5affa58",
-        "RL9": "openhpc-RL9-241022-0038-a5affa58",
-        "RL9-cuda": "openhpc-cuda-RL9-241022-0441-a5affa58"
+        "RL8": "openhpc-RL8-250114-1627-bccc88b5",
+        "RL9": "openhpc-RL9-250114-1626-bccc88b5"
     }
-}
\ No newline at end of file
+}
diff --git a/environments/.stackhpc/terraform/main.tf b/environments/.stackhpc/terraform/main.tf
index ad13ae45d..872003db3 100644
--- a/environments/.stackhpc/terraform/main.tf
+++ b/environments/.stackhpc/terraform/main.tf
@@ -54,6 +54,10 @@ variable "volume_backed_instances" {
     default = false
 }
 
+variable "k3s_token" {
+    type = string
+}
+
 data "openstack_images_image_v2" "cluster" {
     name = var.cluster_image[var.os_version]
     most_recent = true
@@ -69,6 +73,7 @@ module "cluster" {
     key_pair = "slurm-app-ci"
     cluster_image_id = data.openstack_images_image_v2.cluster.id
     control_node_flavor = var.control_node_flavor
+    k3s_token = var.k3s_token
 
     login_nodes = {
         login-0: var.other_node_flavor
@@ -77,6 +82,7 @@ module "cluster" {
         standard: { # NB: can't call this default!
             nodes: ["compute-0", "compute-1"]
             flavor: var.other_node_flavor
+            compute_init_enable: ["compute", "etc_hosts", "nfs", "basic_users", "eessi"]
         }
         # Example of how to add another partition:
         # extra: {
diff --git a/environments/common/inventory/group_vars/all/defaults.yml b/environments/common/inventory/group_vars/all/defaults.yml
index 15340820f..23aafd73e 100644
--- a/environments/common/inventory/group_vars/all/defaults.yml
+++ b/environments/common/inventory/group_vars/all/defaults.yml
@@ -6,6 +6,8 @@ appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}
 appliances_environment_name: "{{ appliances_environment_root | basename | regex_replace('\\W+', '') }}" # [a-zA-Z0-9_] only
 appliances_cockpit_state: absent # RHEL cockpit installed but not enabled in genericcloud images; appliance defaults to removing it
 #appliances_state_dir: # define an absolute path here to use for persistent state: NB: This is defined as /var/lib/state in inventory by the default Terraform
+appliances_mode: configure
+appliances_pulp_url: https://ark.stackhpc.com
 
 # Address(ip/dns) for internal communication between services. This is
 # normally traffic you do no want to expose to users.
@@ -78,4 +80,92 @@ appliances_local_users_default:
 appliances_local_users_extra: [] # see format of appliances_local_users_default above
 appliances_local_users: "{{ appliances_local_users_default + appliances_local_users_extra }}"
 
-###########################################################################################
+################## bootstrap: extra package installs ######################################
+
+appliances_extra_packages_default:
+ - htop
+ - nano
+ - screen
+ - tmux
+ - wget
+ - bind-utils
+ - net-tools
+ - postfix
+ - git
+ - "{{ 'python36' if ansible_distribution_version == '8.9' else 'python312' }}"
+ 
+appliances_extra_packages_other: []
+  
+appliances_extra_packages: "{{ appliances_extra_packages_default + appliances_extra_packages_other }}"
+
+###################### ark repo timestamps ###################################################
+
+appliances_pulp_repos:
+  baseos:
+    '9.4':
+      timestamp: 20241115T011711
+      path: rocky/9.4/BaseOS/x86_64/os
+    '9.5':
+      timestamp: 20241216T013503
+      path: rocky/9.5/BaseOS/x86_64/os
+    '8.10':
+      timestamp: 20241217T123729
+      path: rocky/8.10/BaseOS/x86_64/os
+  appstream:
+    '9.4':
+      timestamp: 20241112T003151
+      path: rocky/9.4/AppStream/x86_64/os
+    '9.5':
+      timestamp: 20241217T005008
+      path: rocky/9.5/AppStream/x86_64/os
+    '8.10':
+      timestamp: 20241217T123729
+      path: rocky/8.10/AppStream/x86_64/os
+  crb:
+    '9.4':
+      timestamp: 20241115T003133
+      path: rocky/9.4/CRB/x86_64/os
+    '9.5':
+      timestamp: 20241217T005008
+      path: rocky/9.5/CRB/x86_64/os
+    '8.10':
+      timestamp: 20241217T123729
+      path: rocky/8.10/PowerTools/x86_64/os
+  extras:
+    '9.4':
+      timestamp: 20241118T002802
+      path: rocky/9.4/extras/x86_64/os
+    '9.5':
+      timestamp: 20241218T004632
+      path: rocky/9.5/extras/x86_64/os
+    '8.10':
+      timestamp: 20241217T123729
+      path: rocky/8.10/extras/x86_64/os
+  epel:
+    '9':
+      timestamp: 20241213T010218
+      path: epel/9/Everything/x86_64
+    '8':
+      timestamp: 20241216T235733
+      path: epel/8/Everything/x86_64
+  openhpc_base:
+    '8':
+      path: OpenHPC/2/EL_8
+      timestamp: 20241218T154614
+    '9':
+      path: OpenHPC/3/EL_9
+      timestamp: 20241218T154614
+  openhpc_updates:
+    '8':
+      path: OpenHPC/2/updates/EL_8
+      timestamp: 20241218T154614
+    '9':
+      path: OpenHPC/3/updates/EL_9
+      timestamp: 20241218T154614
+  ceph:
+    '8':
+      timestamp: 20231104T015751
+      path: centos/8-stream/storage/x86_64/ceph-quincy
+    '9':
+      timestamp: 20240923T233036
+      path: centos/9-stream/storage/x86_64/ceph-reef
diff --git a/environments/common/inventory/group_vars/all/nfs.yml b/environments/common/inventory/group_vars/all/nfs.yml
index bd340b190..45b7c6967 100644
--- a/environments/common/inventory/group_vars/all/nfs.yml
+++ b/environments/common/inventory/group_vars/all/nfs.yml
@@ -3,7 +3,7 @@
 # See: https://github.com/stackhpc/ansible-role-cluster-nfs
 # for variable definitions
 
-nfs_server_default: "{{ hostvars[groups['control'] | first ].internal_address }}"
+nfs_server_default: "{{ groups['control'] | first }}" # avoid using hostvars for compute-init
 
 nfs_configurations:
   - comment: Export /exports/home from Slurm control node as /home
@@ -15,3 +15,9 @@ nfs_configurations:
     nfs_server: "{{ nfs_server_default }}"
     nfs_export: "/exports/home" # assumes skeleton TF is being used
     nfs_client_mnt_point: "/home"
+
+  - comment: Export /exports/cluster from Slurm control node
+    nfs_enable:
+        server: "{{ inventory_hostname in groups['control'] }}"
+        clients: false
+    nfs_export: "/exports/cluster"
diff --git a/environments/common/inventory/group_vars/all/openhpc.yml b/environments/common/inventory/group_vars/all/openhpc.yml
index 6ac1b9ea1..8c981976b 100644
--- a/environments/common/inventory/group_vars/all/openhpc.yml
+++ b/environments/common/inventory/group_vars/all/openhpc.yml
@@ -13,7 +13,7 @@ openhpc_slurm_accounting_storage_type: 'accounting_storage/slurmdbd'
 openhpc_slurmdbd_mysql_database: slurm_acct_db
 openhpc_slurmdbd_mysql_password: "{{ vault_mysql_slurm_password }}"
 openhpc_slurmdbd_mysql_username: slurm
-openhpc_slurm_control_host: "{{ hostvars[groups['control'].0].api_address }}"
+openhpc_slurm_control_host: "{{ groups['control'] | first }}" # avoid using hostvars for compute-init
 openhpc_slurmdbd_host: "{{ openhpc_slurm_control_host }}"
 openhpc_slurm_partitions:
   - name: "compute"
@@ -38,3 +38,16 @@ openhpc_config_default:
 openhpc_config_extra: {}
 openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_extra, list_merge='append') }}"
 openhpc_state_save_location: "{{ appliances_state_dir + '/slurmctld' if appliances_state_dir is defined else '/var/spool' }}"
+
+openhpc_install_type: ohpc # 'ohpc' or 'generic', see https://github.com/stackhpc/ansible-slurm-appliance/pull/326
+
+# Empty repo lists from stackhpc.openhpc role defaults, as these repofiles are
+# now generated by dnf_repos to allow injecting Ark creds:
+ohpc_openhpc_repos:
+  "9": []
+  "8": []
+
+# overriding to ensure doesn't overwrite Ark epel repo
+ohpc_default_extra_repos:
+  "9": []
+  "8": []
diff --git a/environments/common/inventory/group_vars/all/openondemand.yml b/environments/common/inventory/group_vars/all/openondemand.yml
index 5e85392ca..cce923fcc 100644
--- a/environments/common/inventory/group_vars/all/openondemand.yml
+++ b/environments/common/inventory/group_vars/all/openondemand.yml
@@ -5,7 +5,12 @@
 
 # NB: Variables prefixed ood_ are all from https://github.com/OSC/ood-ansible
 
-# openondemand_servername: '' # Must be defined when using openondemand
+openondemand_servername: "{{ hostvars[groups['openondemand'].0].ansible_host if groups['openondemand'] else '' }}"
+
+openondemand_auth: basic_pam
+
+openondemand_jupyter_partition: "{{ openhpc_slurm_partitions[0]['name'] }}"
+openondemand_desktop_partition: "{{ openhpc_slurm_partitions[0]['name'] }}"
 
 # Regex defining hosts which openondemand can proxy; the default regex is compute nodes (for apps) and grafana host,
 # e.g. if the group `compute` has hosts `compute-{0,1,2,..}` this will be '(compute-\d+)|(control)'.
diff --git a/environments/common/inventory/group_vars/all/os-manila-mount.yml b/environments/common/inventory/group_vars/all/os-manila-mount.yml
new file mode 100644
index 000000000..6b25d62cb
--- /dev/null
+++ b/environments/common/inventory/group_vars/all/os-manila-mount.yml
@@ -0,0 +1,3 @@
+# Empty repo lists from stackhpc.ansible-role-os-manila-mount role defaults, as these repofiles are
+# now generated by dnf_repos to allow injecting Ark creds:
+os_manila_mount_ceph_rpm_repos: []
diff --git a/environments/common/inventory/group_vars/all/pulp.yml b/environments/common/inventory/group_vars/all/pulp.yml
new file mode 100644
index 000000000..22bb83216
--- /dev/null
+++ b/environments/common/inventory/group_vars/all/pulp.yml
@@ -0,0 +1,11 @@
+pulp_site_port: 8080
+
+# If using Ark directly (no local Pulp server), override the following with Ark creds
+
+# dnf_repos_username:
+# dnf_repos_password:
+
+# If instead using local Pulp server, override below with Ark creds
+
+# pulp_site_upstream_username:
+# pulp_site_upstream_password:
diff --git a/environments/common/inventory/group_vars/all/selinux.yml b/environments/common/inventory/group_vars/all/selinux.yml
index 25fbbd68f..fef5c3f58 100644
--- a/environments/common/inventory/group_vars/all/selinux.yml
+++ b/environments/common/inventory/group_vars/all/selinux.yml
@@ -1,4 +1,4 @@
 ---
 
-selinux_state: permissive
+selinux_state: disabled
 selinux_policy: targeted
diff --git a/environments/common/inventory/group_vars/all/sshd.yaml b/environments/common/inventory/group_vars/all/sshd.yaml
new file mode 100644
index 000000000..5d4ed228f
--- /dev/null
+++ b/environments/common/inventory/group_vars/all/sshd.yaml
@@ -0,0 +1 @@
+sshd_password_authentication: "{{ sssd_install_ldap | default(false) | bool }}"
diff --git a/environments/common/inventory/group_vars/builder/defaults.yml b/environments/common/inventory/group_vars/builder/defaults.yml
index 22042c1bf..dae4edd9a 100644
--- a/environments/common/inventory/group_vars/builder/defaults.yml
+++ b/environments/common/inventory/group_vars/builder/defaults.yml
@@ -22,3 +22,6 @@ squid_cache_disk: 0 # just needs to be defined
 squid_cache_mem: 0
 tuned_started: false
 tuned_enabled: false
+sssd_started: false
+sssd_enabled: false
+appliances_mode: build
diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups
index 168cc4160..7afad0118 100644
--- a/environments/common/inventory/groups
+++ b/environments/common/inventory/groups
@@ -13,9 +13,6 @@ login
 control
 compute
 
-[eessi:children]
-# Hosts on which EESSI stack should be configured
-
 [hpctests:children]
 # Login group to use for running mpi-based testing.
 login
@@ -79,9 +76,6 @@ cluster
 # Hosts to install firewalld on - see ansible/roles/filewalld
 fail2ban
 
-[block_devices]
-# Superset of hosts to configure filesystems on - see ansible/roles/block_devices/README.md
-
 [basic_users]
 # Add `openhpc` group to add slurm users via creation of users on each node.
 
@@ -118,12 +112,18 @@ freeipa_client
 [cuda]
 # Hosts to install NVIDIA CUDA on - see ansible/roles/cuda/README.md
 
+[eessi]
+# Hosts on which EESSI stack should be configured
+
 [resolv_conf]
 # Allows defining nameservers in /etc/resolv.conf - see ansible/roles/resolv_conf/README.md
 
 [proxy]
 # Hosts to configure http/s proxies - see ansible/roles/proxy/README.md
 
+[manila]
+# Hosts to configure for manila fileshares
+
 [persist_hostkeys]
 # Hosts to persist hostkeys for across reimaging. NB: Requires appliances_state_dir on hosts.
 
@@ -135,3 +135,33 @@ freeipa_client
 
 [ansible_init]
 # Hosts to run linux-anisble-init
+
+[sssd]
+# Hosts to configure sssd on
+
+[sshd]
+# Hosts where the OpenSSH server daemon should be configured
+
+[compute_init]
+# EXPERIMENTAL: Compute hosts to enable joining cluster on boot on
+
+[k3s]
+# Hosts to run k3s server/agent
+
+[k9s]
+# Hosts to install k9s on
+
+[lustre]
+# Hosts to run lustre client
+
+[extra_packages]
+# Hosts to install specified additional packages on
+
+[dnf_repos:children]
+# Hosts to replace system repos with Pulp repos
+# Warning: when using Ark directly rather than a local Pulp server, adding hosts other than `builder` will leak Ark creds to users
+builder
+extra_packages
+
+[pulp]
+# Add builder to this group to enable automatically syncing of pulp during image build
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
index 9707f9c9c..27811f734 100644
--- a/environments/common/layouts/everything
+++ b/environments/common/layouts/everything
@@ -36,8 +36,9 @@ login
 [block_devices:children]
 # Environment-specific so not defined here
 
-[basic_users]
+[basic_users:children]
 # Add `openhpc` group to add Slurm users via creation of users on each node.
+openhpc
 
 # [openondemand:children]
 # # Host to run Open Ondemand server on - subset of login
@@ -51,13 +52,15 @@ login
 # Subset of compute to run a Jupyter Notebook servers on via Open Ondemand
 compute
 
-[etc_hosts]
+[etc_hosts:children]
 # Hosts to manage /etc/hosts e.g. if no internal DNS. See ansible/roles/etc_hosts/README.md
+cluster
 
 [cuda]
 # Hosts to install NVIDIA CUDA on - see ansible/roles/cuda/README.md
 
 [eessi:children]
+# Hosts on which EESSI stack should be configured
 openhpc
 
 [resolv_conf]
@@ -69,8 +72,10 @@ openhpc
 [manila]
 # Hosts to configure for manila fileshares
 
-[persist_hostkeys]
-# Hosts to persist hostkeys for across reimaging. NB: Requires appliances_state_dir on hosts.
+[persist_hostkeys:children]
+# Hosts to use common set of hostkeys which persist across reimaging.
+login
+openondemand
 
 [squid]
 # Hosts to run squid proxy
@@ -79,5 +84,30 @@ openhpc
 # Hosts to run TuneD configuration
 
 [ansible_init:children]
-# Hosts to run ansible-init
-cluster
\ No newline at end of file
+# Hosts to run linux-anisble-init
+cluster
+
+[sssd]
+# Hosts to configure sssd on
+
+[sshd]
+# Hosts where the OpenSSH server daemon should be configured
+
+[compute_init:children]
+# EXPERIMENTAL: Compute hosts to enable joining cluster on boot on
+compute
+
+[k3s:children]
+# Hosts to run k3s server/agent
+openhpc
+
+[k9s:children]
+# Hosts to install k9s on
+control
+
+[lustre]
+# Hosts to run lustre client
+
+[extra_packages:children]
+# Hosts to install specified additional packages on
+builder
diff --git a/environments/nrel/hooks/pre.yml b/environments/nrel/hooks/pre.yml
index cb0c1b0d8..fa1d2eb55 100644
--- a/environments/nrel/hooks/pre.yml
+++ b/environments/nrel/hooks/pre.yml
@@ -36,39 +36,6 @@
   become: true
   gather_facts: false
 
-  # tasks:
-  #   - name: Does home/USER dir exist
-  #     ansible.builtin.stat:
-  #       path: "/home/{{ appliances_local_users_ansible_user_name }}"
-  #     register: homerocky
-
-  #   - name: Does var/lib/rocky dir exist
-  #     ansible.builtin.stat:
-  #       path: "/var/lib/{{ appliances_local_users_ansible_user_name }}"
-  #     register: varlibrocky
-
-  #   - name: Cp homerocky to var/lib/ dir
-  #     ansible.builtin.command: cp -a /home/{{ appliances_local_users_ansible_user_name }} /var/lib/{{ appliances_local_users_ansible_user_name }}
-  #     when:
-  #       - homerocky.stat.exists
-  #     register: cph
-  #     changed_when: cph.rc != 0
-
-  #   # - name: cp auth_keys
-  #   #   shell: |
-  #   #     cp -a /home/rocky/.ssh /var/lib/rocky/.ssh
-  #   #   when:
-  #   #     - varlibrocky.stat.exists
-  #   #     - homerocky.stat.exists
-
-  #   - name: Hack passwd file for localhome/rocky
-  #     ansible.builtin.lineinfile:
-  #       path: /etc/passwd
-  #       regexp: '^rocky.*1000.*'
-  #       line: "rocky:x:1000:1000:vsRockyUser:/var/lib/{{ appliances_local_users_ansible_user_name }}:/bin/bash"
-  # tags:
-  #   - rocky_localhome
-
 - name: "Do the preliminary node setups"
   hosts: all
   # tags: pre_tasks_all
diff --git a/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/basic_users.yml b/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/basic_users.yml
new file mode 100644
index 000000000..dc993c3b8
--- /dev/null
+++ b/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/basic_users.yml
@@ -0,0 +1,4 @@
+basic_users_users:
+  - name: demo_user
+    password: "{% raw %}{{ vault_demo_user_password | password_hash('sha512', 65534 | random(seed=inventory_hostname) | string) }}{% endraw %}" # idempotent
+    uid: 1005
diff --git a/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/grafana.yml b/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/grafana.yml
new file mode 100644
index 000000000..521616a1b
--- /dev/null
+++ b/environments/skeleton/{{cookiecutter.environment}}/inventory/group_vars/all/grafana.yml
@@ -0,0 +1 @@
+grafana_auth_anonymous: true
\ No newline at end of file
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/compute.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/compute.tf
index c8907c836..a90108924 100644
--- a/environments/skeleton/{{cookiecutter.environment}}/terraform/compute.tf
+++ b/environments/skeleton/{{cookiecutter.environment}}/terraform/compute.tf
@@ -3,17 +3,28 @@ module "compute" {
 
   for_each = var.compute
 
+  # must be set for group:
   nodes = each.value.nodes
+  flavor = each.value.flavor
+
   cluster_name = var.cluster_name
   cluster_domain_suffix = var.cluster_domain_suffix
   cluster_net_id = data.openstack_networking_network_v2.cluster_net.id
   cluster_subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id
 
-  flavor = each.value.flavor
+  # can be set for group, defaults to top-level value:
   image_id = lookup(each.value, "image_id", var.cluster_image_id)
   vnic_type = lookup(each.value, "vnic_type", var.vnic_type)
   vnic_profile = lookup(each.value, "vnic_profile", var.vnic_profile)
+  volume_backed_instances = lookup(each.value, "volume_backed_instances", var.volume_backed_instances)
+  root_volume_size = lookup(each.value, "root_volume_size", var.root_volume_size)
+  extra_volumes = lookup(each.value, "extra_volumes", {})
+
+  compute_init_enable = lookup(each.value, "compute_init_enable", [])
+
   key_pair = var.key_pair
   environment_root = var.environment_root
+  k3s_token = var.k3s_token
+  control_address = [for n in openstack_compute_instance_v2.control["control"].network: n.fixed_ip_v4 if n.access_network][0]
   security_group_ids = [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id]
 }
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/nodes.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/nodes.tf
index 006f802c7..9bb75466e 100644
--- a/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/nodes.tf
+++ b/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/nodes.tf
@@ -1,3 +1,33 @@
+locals {
+  all_compute_volumes = {for v in setproduct(var.nodes, keys(var.extra_volumes)): "${v[0]}-${v[1]}" => {"node" = v[0], "volume" = v[1]}}
+  # e.g. with
+  # var.nodes = ["compute-0", "compute-1"]
+  # var.extra_volumes = {
+  #     "vol-a" = {size = 10},
+  #     "vol-b" = {size = 20}
+  # }
+  # this is a mapping with
+  # keys "compute-0-vol-a", "compute-0-vol-b" ...
+  # values which are a mapping e.g. {"node"="compute-0", "volume"="vol-a"}
+}
+
+resource "openstack_blockstorage_volume_v3" "compute" {
+
+    for_each = local.all_compute_volumes
+
+    name = "${var.cluster_name}-${each.key}"
+    description = "Compute node ${each.value.node} volume ${each.value.volume}"
+    size = var.extra_volumes[each.value.volume].size
+}
+
+resource "openstack_compute_volume_attach_v2" "compute" {
+
+  for_each = local.all_compute_volumes
+
+  instance_id = openstack_compute_instance_v2.compute["${each.value.node}"].id
+  volume_id  = openstack_blockstorage_volume_v3.compute["${each.key}"].id
+}
+
 resource "openstack_networking_port_v2" "compute" {
 
   for_each = toset(var.nodes)
@@ -44,9 +74,14 @@ resource "openstack_compute_instance_v2" "compute" {
     access_network = true
   }
 
-  metadata = {
-    environment_root = var.environment_root
-  }
+  metadata = merge(
+    {
+        environment_root = var.environment_root
+        k3s_token          = var.k3s_token
+        control_address    = var.control_address
+     },
+    {for e in var.compute_init_enable: e => true}
+  )
 
   user_data = <<-EOF
     #cloud-config
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/variables.tf
index 5696a94eb..b0e489017 100644
--- a/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/variables.tf
+++ b/environments/skeleton/{{cookiecutter.environment}}/terraform/compute/variables.tf
@@ -64,6 +64,33 @@ variable "root_volume_size" {
     default = 40
 }
 
+variable "extra_volumes" {
+    description = <<-EOF
+        Mapping defining additional volumes to create and attach.
+        Keys are unique volume name.
+        Values are a mapping with:
+            size: Size of volume in GB
+        **NB**: The order in /dev is not guaranteed to match the mapping
+        EOF
+    type = any
+    default = {}
+}
+
 variable "security_group_ids" {
     type = list
 }
+
+variable "k3s_token" {
+    type = string
+}
+
+variable "control_address" {
+    description = "Name/address of control node"
+    type = string
+}
+
+variable "compute_init_enable" {
+    type = list(string)
+    description = "Groups to activate for ansible-init compute rebuilds"
+    default = []
+}
\ No newline at end of file
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf
index 376d3da0e..8ea8cabcb 100644
--- a/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf
+++ b/environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf
@@ -76,6 +76,7 @@ resource "openstack_compute_instance_v2" "control" {
 
   metadata = {
     environment_root = var.environment_root
+    k3s_token = var.k3s_token
   }
 
   user_data = <<-EOF
@@ -124,6 +125,8 @@ resource "openstack_compute_instance_v2" "login" {
 
   metadata = {
     environment_root = var.environment_root
+    k3s_token = var.k3s_token
+    control_address = [for n in openstack_compute_instance_v2.control["control"].network: n.fixed_ip_v4 if n.access_network][0]
   }
 
   user_data = <<-EOF
diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf b/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf
index 289de3fef..bdffd40ce 100644
--- a/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf
+++ b/environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf
@@ -6,7 +6,7 @@ variable "cluster_name" {
 variable "cluster_domain_suffix" {
     type = string
     description = "Domain suffix for cluster"
-    default = "invalid"
+    default = "internal"
 }
 
 variable "cluster_net" {
@@ -52,6 +52,14 @@ variable "compute" {
             image_id: Overrides variable cluster_image_id
             vnic_type: Overrides variable vnic_type
             vnic_profile: Overrides variable vnic_profile
+            compute_init_enable: Toggles compute-init rebuild (see compute-init role docs)
+            volume_backed_instances: Overrides variable volume_backed_instances
+            root_volume_size: Overrides variable root_volume_size
+            extra_volumes: Mapping defining additional volumes to create and attach
+                           Keys are unique volume name.
+                           Values are a mapping with:
+                                size: Size of volume in GB
+                           **NB**: The order in /dev is not guaranteed to match the mapping
     EOF
 }
 
@@ -131,3 +139,8 @@ variable "root_volume_size" {
     type = number
     default = 40
 }
+
+variable "k3s_token" {
+    description = "K3s cluster authentication token, set automatically by Ansible"
+    type = string
+}
\ No newline at end of file
diff --git a/packer/README.md b/packer/README.md
deleted file mode 100644
index 5e1d57dc2..000000000
--- a/packer/README.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# Packer-based image build
-
-The appliance contains code and configuration to use [Packer](https://developer.hashicorp.com/packer) with the [OpenStack builder](https://www.packer.io/plugins/builders/openstack) to build images.
-
-The Packer configuration defined here builds "fat images" which contain binaries for all nodes, but no cluster-specific configuration. Using these:
-- Enables the image to be tested in CI before production use.
-- Ensures re-deployment of the cluster or deployment of additional nodes can be completed even if packages are changed in upstream repositories (e.g. due to RockyLinux or OpenHPC updates).
-- Improves deployment speed by reducing the number of package downloads to improve deployment speed.
-
-By default, a fat image build starts from a nightly image build containing Mellanox OFED, and updates all DNF packages already present. The 'latest' nightly build itself is from a RockyLinux GenericCloud image.
-
-The fat images StackHPC builds and test in CI are available from [GitHub releases](https://github.com/stackhpc/ansible-slurm-appliance/releases). However with some additional configuration it is also possible to:
-1. Build site-specific fat images from scratch.
-2. Extend an existing fat image with additional software.
-
-
-# Usage
-
-The steps for building site-specific fat images or extending an existing fat image are the same:
-
-1. Ensure the current OpenStack credentials have sufficient authorisation to upload images (this may or may not require the `member` role for an application credential, depending on your OpenStack configuration).
-2. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments/<environment>/builder.pkrvars.hcl` containing at a minimum e.g.:
-  
-    ```hcl
-    flavor = "general.v1.small"                           # VM flavor to use for builder VMs
-    networks = ["26023e3d-bc8e-459c-8def-dbd47ab01756"]   # List of network UUIDs to attach the VM to
-    ```
-    
-    - The network used for the Packer VM must provide outbound internet access but does not need to provide access to resources which the final cluster nodes require (e.g. Slurm control node, network filesystem servers etc.).
-    
-    - For additional options such as non-default private key locations or jumphost configuration see the variable descriptions in `./openstack.pkr.hcl`.
-
-    - For an example of configuration for extending an existing fat image see below.
-
-3. Activate the venv and the relevant environment.
-
-4. Build images using the relevant variable definition file, e.g.:
-
-        cd packer/
-        PACKER_LOG=1 /usr/bin/packer build -only=openstack.openhpc --on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
-
-  Note that the `-only` flag here restricts the build to the non-CUDA fat image "source" (in Packer terminology). Other
-  source options are:
-    - `-only=openstack.openhpc-cuda`: Build a fat image including CUDA packages.
-    - `-only=openstack.openhpc-extra`: Build an image which extends an existing fat image - in this case the variable `source_image` or `source_image_name}` must also be set in the Packer variables file.
-    
-5. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc-` and including a timestamp and a shortened git hash.
-
-# Build Process
-
-In summary, Packer creates an OpenStack VM, runs Ansible on that, shuts it down, then creates an image from the root disk.
-
-Many of the Packer variables defined in `openstack.pkr.hcl` control the definition of the build VM and how to SSH to it to run Ansible, which are generic OpenStack builder options. Packer varibles can be set in a file at any convenient path; the above
-example shows the use of the environment variable `$PKR_VAR_environment_root` (which itself sets the Packer variable
-`environment_root`) to automatically select a variable file from the current environment, but for site-specific builds
-using a path in a "parent" environment is likely to be more appropriate (as builds should not be environment-specific, to allow testing).
-
-What is Slurm Appliance-specific are the details of how Ansible is run:
-- The build VM is always added to the `builder` inventory group, which differentiates it from "real" nodes. This allows
-  variables to be set differently during Packer builds, e.g. to prevent services starting. The defaults for this are in `environments/common/inventory/group_vars/builder/`, which could be extended or overriden for site-specific fat image builds using `builder` groupvars for the relevant environment. It also runs some builder-specific code (e.g. to ensure Packer's SSH
-  keys are removed from the image).
-- The default fat image build also adds the build VM to the "top-level" `compute`, `control` and `login` groups. This ensures
-  the Ansible specific to all of these types of nodes run (other inventory groups are constructed from these by `environments/common/inventory/groups file` - this is not builder-specific).
-- Which groups the build VM is added to is controlled by the Packer `groups` variable. This can be redefined for builds using the `openhpc-extra` source to add the build VM into specific groups. E.g. with a Packer variable file:
-
-      source_image_name = {
-          RL9 = "openhpc-ofed-RL9-240619-0949-66c0e540"
-      }
-      groups = {
-          openhpc-extra = ["foo"]
-      }
-
-    the build VM uses an existing "fat image" (rather than a 'latest' nightly one) and is added to the `builder` and `foo` groups. This means only code targeting `builder` and `foo` groups runs. In this way an existing image can be extended with site-specific code, without modifying the part of the image which has already been tested in the StackHPC CI.
-
- - The playbook `ansible/fatimage.yml` is run which is only a subset of `ansible/site.yml`. This allows restricting the code
-   which runs during build for cases where setting `builder` groupvars is not sufficient (e.g. a role always attempts to configure or start services). This may eventually be removed.
-
-There are some things to be aware of when developing Ansible to run in a Packer build VM:
-  - Only some tasks make sense. E.g. any services with a reliance on the network cannot be started, and may not be able to be enabled if when creating an instance with the resulting image the remote service will not be immediately present.
-  - Nothing should be written to the persistent state directory `appliances_state_dir`, as this is on the root filesystem rather than an OpenStack volume.
-  - Care should be taken not to leave data on the root filesystem which is not wanted in the final image, (e.g secrets).
-  - Build VM hostnames are not the same as for equivalent "real" hosts and do not contain `login`, `control` etc. Therefore variables used by the build VM must be defined as groupvars not hostvars.
-  - Ansible may need to proxy to real compute nodes. If Packer should not use the same proxy to connect to the
-    build VMs (e.g. build happens on a different network), proxy configuration should not be added to the `all` group.
-  - Currently two fat image "sources" are defined, with and without CUDA. This simplifies CI configuration by allowing the
-    default source images to be defined in the `openstack.pkr.hcl` definition.
diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl
index cd18ab3cd..817c9402a 100644
--- a/packer/openstack.pkr.hcl
+++ b/packer/openstack.pkr.hcl
@@ -23,6 +23,7 @@ data "git-commit" "cwd-head" { }
 locals {
     git_commit = data.git-commit.cwd-head.hash
     timestamp = formatdate("YYMMDD-hhmm", timestamp())
+    image_name_version = var.image_name_version == "auto" ? "-${local.timestamp}-${substr(local.git_commit, 0, 8)}" : var.image_name_version
 }
 
 # Path pointing to root of repository - automatically set by environment variable PKR_VAR_repo_root
@@ -39,15 +40,10 @@ variable "networks" {
   type = list(string)
 }
 
-variable "os_version" {
-  type = string
-  description = "'RL8' or 'RL9' with default source_image_* mappings"
-  default = "RL9"
-}
-
 # Must supply either source_image_name or source_image_id
 variable "source_image_name" {
   type = string
+  default = null
   description = "name of source image"
 }
 
@@ -122,14 +118,8 @@ variable "volume_type" {
 }
 
 variable "volume_size" {
-  type = map(number)
-  default = {
-    # fat image builds, GB:
-    rocky-latest = 30
-    rocky-latest-cuda = 30
-    openhpc = 30
-    openhpc-cuda = 30
-  }
+  type = number
+  default = 15
 }
 
 variable "image_disk_format" {
@@ -142,16 +132,22 @@ variable "metadata" {
   default = {}
 }
 
-variable "groups" {
-  type = map(list(string))
-  description = "Additional inventory groups (other than 'builder') to add build VM to, keyed by source name"
-  default = {
-    # fat image builds:
-    rocky-latest = ["update", "ofed"]
-    rocky-latest-cuda = ["update", "ofed", "cuda"]
-    openhpc = ["control", "compute", "login"]
-    openhpc-cuda = ["control", "compute", "login"]
-  }
+variable "inventory_groups" {
+  type = string
+  description = "Comma-separated list of additional inventory groups (other than 'builder') to add build VM to. Default is none."
+  default = ""
+}
+
+variable "image_name" {
+  type = string
+  description = "Name of image"
+  default = "openhpc"
+}
+
+variable "image_name_version" {
+  type = string
+  description = "Suffix for image name giving version. Default of 'auto' appends timestamp + short commit"
+  default = "auto"
 }
 
 source "openstack" "openhpc" {
@@ -159,9 +155,11 @@ source "openstack" "openhpc" {
   flavor = var.flavor
   use_blockstorage_volume = var.use_blockstorage_volume
   volume_type = var.volume_type
-  volume_size = var.volume_size[source.name]
+  volume_size = var.volume_size
   metadata = var.metadata
-  instance_metadata = {ansible_init_disable = "true"}
+  instance_metadata = {
+    ansible_init_disable = "true"
+  }
   networks = var.networks
   floating_ip_network = var.floating_ip_network
   security_groups = var.security_groups
@@ -187,39 +185,13 @@ source "openstack" "openhpc" {
 
 build {
 
-  # latest nightly image:
-  source "source.openstack.openhpc" {
-    name = "rocky-latest"
-    image_name = "${source.name}-${var.os_version}"
-  }
-
-  # latest nightly cuda image:
-  source "source.openstack.openhpc" {
-    name = "rocky-latest-cuda"
-    image_name = "${source.name}-${var.os_version}"
-  }
-
-  # OFED fat image:
-  source "source.openstack.openhpc" {
-    name = "openhpc"
-    image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}"
-  }
-
-  # CUDA fat image:
-  source "source.openstack.openhpc" {
-    name = "openhpc-cuda"
-    image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}"
-  }
-
-  # Extended site-specific image, built on fat image:
   source "source.openstack.openhpc" {
-    name = "openhpc-extra"
-    image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}"
+    image_name = "${var.image_name}${local.image_name_version}"
   }
 
   provisioner "ansible" {
     playbook_file = "${var.repo_root}/ansible/fatimage.yml"
-    groups = concat(["builder"], var.groups[source.name])
+    groups = concat(["builder"], var.inventory_groups == "" ? [] : split(",", var.inventory_groups))
     keep_inventory_file = true # for debugging
     use_proxy = false # see https://www.packer.io/docs/provisioners/ansible#troubleshooting
     extra_arguments = [
diff --git a/requirements.txt b/requirements.txt
index 6651506fb..872ee9516 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-ansible==6.0.0
+ansible==6.7.0 # cloudalchemy.prometheus uses ansible.builtin.include, removed in ansible-core==2.16 => ansible==9
 openstacksdk
 python-openstackclient==6.6.1 # v7.0.0 has a bug re. rebuild
 python-manilaclient
@@ -9,3 +9,4 @@ cookiecutter
 selinux # this is a shim to avoid having to use --system-site-packages, you still need sudo yum install libselinux-python3
 netaddr
 matplotlib
+pulp-cli==0.23.2
diff --git a/requirements.yml b/requirements.yml
index 8b3fbcdd3..e06c60b75 100644
--- a/requirements.yml
+++ b/requirements.yml
@@ -21,11 +21,11 @@ roles:
     version: v3.1.5
   - src: https://github.com/stackhpc/ansible-role-os-manila-mount.git
     name: stackhpc.os-manila-mount
-    version: v24.5.1 # Support ceph quincy for RL9
+    version: v25.1.1
 
 collections:
   - name: containers.podman
-    version: 1.10.2
+    version: 1.16.2
   - name: community.grafana
     version: 1.5.4
   - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools
@@ -49,4 +49,10 @@ collections:
   - name: https://github.com/azimuth-cloud/ansible-collection-image-utils
     type: git
     version: 0.4.0
+  # stackhpc.pulp has pulp.squeezer as dependency, any version, but latest
+  # requires newer ansible than can install
+  - name: pulp.squeezer
+    version: 0.0.15
+  - name: stackhpc.pulp
+    version: 0.5.5
 ...