Improve docs and fix script

Author: jovial

ansible/fatimage.yml

@@ -261,15 +261,18 @@
     - name: Recompile and install slurm packages
       shell: |
         #!/bin/bash
-        dnf download --source slurm-slurmd-ohpc
+        set -eux
+        dnf download -y --source slurm-slurmd-ohpc
         rpm -i slurm-ohpc-*.src.rpm
         dnf install -y  @'Development Tools'
         cd /root/rpmbuild/SPECS
         dnf builddep -y slurm.spec
-        rpmbuild -bb -D "_with_nvml --with-nvml=/usr/local/cuda-{{ cuda_facts_version_short }}/targets/x86_64-linux/"
-        dnf reinstall /root/rpmbuild/RPMS/x86_64/*.rpm
+        rpmbuild -bb -D "_with_nvml --with-nvml=/usr/local/cuda-{{ cuda_facts_version_short }}/targets/x86_64-linux/" slurm.spec
+        dnf reinstall -y /root/rpmbuild/RPMS/x86_64/*.rpm
         # Workaround path issue: https://groups.google.com/g/slurm-users/c/cvGb4JnK8BY
-        ln -s /lib64/libnvidia-ml.so.1 /lib64/libnvidia-ml.so
+        if [[ -e /lib64/libnvidia-ml.so ]]; then
+          ln -s /lib64/libnvidia-ml.so.1 /lib64/libnvidia-ml.so
+        fi
 
 - name: Run post.yml hook
   vars:

docs/mig.md

@@ -26,8 +26,12 @@ For example in: `environments/<environment>/inventory/group_vars/all/vgpu`:
 vgpu_definitions:
     - pci_address: "0000:17:00.0"
       mig_devices:
-        "1g.10gb": 1
-        "2g.20gb": 3
+        "1g.10gb": 4
+        "4g.40gb": 1
+    - pci_address: "0000:81:00.0"
+      mig_devices:
+        "1g.10gb": 4
+        "4g.40gb": 1
 ```
 
 The appliance will use the driver installed via the ``cuda`` role. Use ``lspci`` to determine the PCI
@@ -39,4 +43,18 @@ Use the ``vgpu`` metadata option to enable creation of mig devices on rebuild.
 
 ## gres configuration
 
-TODO
+You should stop terraform templating out partitions.yml and specify `openhpc_slurm_partitions` manually.
+An example enabling the nvml autodection mechanism is show below
+(`environments/<environment>/inventory/group_vars/all/partitions-manual.yml`):
+
+```
+openhpc_slurm_partitions:
+    - name: cpu
+    - name: gpu
+      gres_autodetect: nvml
+      gres:
+        # Two cards not partitioned with MIG
+        - conf: "gpu:nvidia_h100_80gb_hbm3:2"
+        - conf: "gpu:nvidia_h100_80gb_hbm3_4g.40gb:2"
+        - conf: "gpu:nvidia_h100_80gb_hbm3_1g.10gb:6"
+```