Merge pull request #253 from stackhpc/cuda

Add support for CUDA

Author: sjpb

ansible/.gitignore

@@ -34,6 +34,8 @@ roles/*
 !roles/mysql/**
 !roles/systemd/
 !roles/systemd/**
+!roles/cuda/
+!roles/cuda/**
 !roles/freeipa/
 !roles/freeipa/**
 !roles/proxy/

ansible/adhoc/cudatests.yml

@@ -0,0 +1,8 @@
+- hosts: cuda
+  become: yes
+  gather_facts: no
+  tags: cuda_samples
+  tasks:
+    - import_role:
+        name: cuda
+        tasks_from: samples.yml

ansible/extras.yml

@@ -0,0 +1,7 @@
+- hosts: cuda
+  become: yes
+  gather_facts: no
+  tags: cuda
+  tasks:
+    - import_role:
+        name: cuda

ansible/fatimage.yml

@@ -32,6 +32,12 @@
       dnf:
         name: nfs-utils
 
+- import_playbook: extras.yml
+
+- hosts: builder
+  become: yes
+  gather_facts: no
+  tasks:
     # - import_playbook: slurm.yml
     - name: OpenHPC
       import_role:

ansible/roles/cuda/README.md

@@ -0,0 +1,13 @@
+# cuda
+
+Install NVIDIA CUDA. The CUDA binaries are added to the PATH for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
+
+## Prerequisites
+
+Requires OFED to be installed to provide required kernel-* packages.
+
+## Role Variables
+
+- `cuda_distro`: Optional. Default `rhel8`.
+- `cuda_repo`: Optional. Default `https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo`
+- `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds']`.

ansible/roles/cuda/defaults/main.yml

@@ -0,0 +1,13 @@
+cuda_distro: rhel8
+cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
+cuda_packages:
+  - cuda
+  - nvidia-gds
+# cuda_version_tuple: # discovered from installed package e.g. ('12', '1', '0')
+cuda_version_short: "{{ cuda_version_tuple[0] }}.{{ cuda_version_tuple[1] }}"
+cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
+cuda_samples_path: "/home/{{ ansible_user }}/cuda_samples"
+cuda_samples_programs:
+  - deviceQuery
+  - bandwidthTest
+# cuda_devices: # discovered from deviceQuery run

ansible/roles/cuda/tasks/main.yml

@@ -0,0 +1,53 @@
+
+# Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#redhat8-installation
+
+- name: Check for OFED
+  command:
+    cmd: dnf list --installed rdma-core
+  register: _dnf_rdma_core
+  changed_when: false
+
+- name: Assert OFED installed
+  assert: 
+    that: "'mlnx' in _dnf_rdma_core.stdout"
+    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED installed?"
+
+- name: Install cuda repo
+  get_url:
+    dest: "/etc/yum.repos.d/cuda-{{ cuda_distro }}.repo"
+    url: "{{ cuda_repo }}"
+
+- name: Enable nvidia driver module
+  ansible.builtin.command: dnf module enable -y nvidia-driver:latest-dkms
+  register: nvidiadriver_enable
+  changed_when: "'Nothing to do' not in nvidiadriver_enable.stdout"
+
+- name: Install nvidia driver module
+  ansible.builtin.command: dnf module install -y nvidia-driver:latest-dkms
+  register: nvidiadriver_install
+  changed_when: "'Nothing to do' not in nvidiadriver_install.stdout"
+
+- name: Install cuda packages
+  ansible.builtin.dnf:
+    name: "{{ cuda_packages }}"
+  register: cuda_package_install
+
+- name: Add latest cuda binaries to path
+  lineinfile:
+    path: /etc/profile.d/sh.local
+    line: 'export PATH=$PATH:$(ls -1d /usr/local/cuda-* | sort -V | tail -1)/bin'
+
+- name: Enable NVIDIA Persistence Daemon
+  systemd:
+    name: nvidia-persistenced
+    enabled: true
+    state: started
+
+- name: Reboot
+  ansible.builtin.reboot:
+    post_reboot_delay: 30
+  when: cuda_package_install.changed
+
+- name: Wait for hosts to be reachable
+  wait_for_connection:
+    sleep: 15

ansible/roles/cuda/tasks/samples.yml

@@ -0,0 +1,63 @@
+- name: Read cuda version file
+  slurp:
+    src: /usr/local/cuda/version.json
+  register: _cuda_samples_version
+
+- name: Set fact for discovered cuda version
+  set_fact:
+    cuda_version_tuple: "{{ (_cuda_samples_version.content | b64decode | from_json).cuda.version | split('.') }}" # e.g. '12.1.0'
+
+- name: Ensure cuda_samples_path exists
+  file:
+    state: directory
+    path: "{{ cuda_samples_path }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+
+- name: Download cuda sample release
+  unarchive:
+    remote_src: yes
+    src: "{{ cuda_samples_release_url }}"
+    dest: "{{ cuda_samples_path }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+
+- name: Build cuda samples
+  shell:
+    cmd: make
+    chdir: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/Samples/1_Utilities/{{ item }}"
+    creates: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/bin/x86_64/linux/release/{{ item }}"
+  loop: "{{ cuda_samples_programs }}"
+
+- name: Run cuda deviceQuery
+  command:
+    cmd: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/bin/x86_64/linux/release/deviceQuery"
+  register: _cuda_devicequery
+
+- name: Set fact for cuda devices
+  set_fact:
+    cuda_devices: "{{ _cuda_devicequery.stdout | regex_findall('Device (\\d+):') }}"
+
+- name: Run cuda bandwidth test
+  command:
+    cmd: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/bin/x86_64/linux/release/bandwidthTest --device={{ item }}"
+  register: _cuda_bandwidthtest
+  loop: "{{ cuda_devices }}"
+  loop_control:
+    label: "Device {{ item }}" # e.g '0'
+
+- name: Summarise bandwidth test output
+  debug:
+    msg: |
+      {{ _parts[1].splitlines()[0] | trim }}
+      Bandwidths: (Gb/s)
+      Host to Device: {{ _parts[2].split()[-1] }}
+      Device to Host: {{ _parts[3].split()[-1] }}
+      Device to Device: {{ _parts[4].split()[-1] }}
+      {{ ': '.join(_parts[5].split('=') | map('trim')) }}
+      {{ _parts[6] }}
+  loop: "{{ _cuda_bandwidthtest.results }}"
+  vars:
+    _parts: "{{ item.stdout.split('\n\n') }}"
+  loop_control:
+    label: "Device {{ item.item }}" # e.g '0'

ansible/site.yml

@@ -18,6 +18,7 @@
   when: hook_path | exists
 
 - import_playbook: filesystems.yml
+- import_playbook: extras.yml
 - import_playbook: slurm.yml
 - import_playbook: portal.yml
 - import_playbook: monitoring.yml

environments/common/inventory/groups

@@ -104,6 +104,8 @@ grafana
 control
 prometheus
 
+[cuda]
+# Hosts to install NVIDIA CUDA on - see ansible/roles/cuda/README.md
 [resolv_conf]
 # Allows defining nameservers in /etc/resolv.conf - see ansible/roles/resolv_conf/README.md