Initialised github repo to buil our own Nvidia driver images for the DSRI

Based on the following Nvidia repo: https://gitlab.com/nvidia/container-images/driver

Author: lwinckers

Dockerfile

@@ -0,0 +1,93 @@
+FROM nvcr.io/nvidia/cuda:12.9.1-base-ubi8 as license
+
+# Build nvidia-container-runtime binary
+FROM golang:1.24.4 as build-runtime
+
+WORKDIR /go/src/nvidia-container-runtime
+COPY nvidia-container-runtime .
+RUN go install -v nvidia-container-runtime
+
+# Build driver image
+FROM ubuntu:16.04
+
+RUN dpkg --add-architecture i386 && \
+    apt-get update && apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        apt-utils \
+	bc \
+	binutils \
+        build-essential \
+        ca-certificates \
+        curl \
+	gnupg2 \
+	jq \
+        kmod \
+        libc6:i386 \
+        libelf-dev \
+	libssl-dev \
+	module-init-tools \
+	software-properties-common && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic main" > /etc/apt/sources.list && \
+    echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates main" >> /etc/apt/sources.list && \
+    echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security main" >> /etc/apt/sources.list && \
+    usermod -o -u 0 -g 0 _apt
+
+RUN curl -fsSL -o /usr/local/bin/donkey https://github.com/3XX0/donkey/releases/download/v1.1.0/donkey && \
+    curl -fsSL -o /usr/local/bin/extract-vmlinux https://raw.githubusercontent.com/torvalds/linux/master/scripts/extract-vmlinux && \
+    chmod +x /usr/local/bin/donkey /usr/local/bin/extract-vmlinux
+
+#ARG BASE_URL=http://us.download.nvidia.com/XFree86/Linux-x86_64
+ARG BASE_URL=https://us.download.nvidia.com/tesla
+ARG DRIVER_VERSION=450.80.02
+ENV DRIVER_VERSION=$DRIVER_VERSION
+
+# Install the userspace components and copy the kernel module sources.
+RUN cd /tmp && \
+    curl -fSsl -O $BASE_URL/$DRIVER_VERSION/NVIDIA-Linux-x86_64-$DRIVER_VERSION.run && \
+    sh NVIDIA-Linux-x86_64-$DRIVER_VERSION.run -x && \
+    cd NVIDIA-Linux-x86_64-$DRIVER_VERSION* && \
+    ./nvidia-installer --silent \
+                       --no-kernel-module \
+                       --install-compat32-libs \
+                       --no-nouveau-check \
+                       --no-nvidia-modprobe \
+                       --no-rpms \
+                       --no-backup \
+                       --no-check-for-alternate-installs \
+                       --no-libglx-indirect \
+                       --no-install-libglvnd \
+                       --x-prefix=/tmp/null \
+                       --x-module-path=/tmp/null \
+                       --x-library-path=/tmp/null \
+                       --x-sysconfig-path=/tmp/null && \
+    mkdir -p /usr/src/nvidia-$DRIVER_VERSION && \
+    mv LICENSE mkprecompiled kernel /usr/src/nvidia-$DRIVER_VERSION && \
+    sed '9,${/^\(kernel\|LICENSE\)/!d}' .manifest > /usr/src/nvidia-$DRIVER_VERSION/.manifest && \
+    rm -rf /tmp/*
+
+# Install and configure nvidia-container-runtime
+ENV NVIDIA_VISIBLE_DEVICES void
+
+COPY --from=build-runtime /go/bin/nvidia-container-runtime /usr/bin/nvidia-container-runtime
+
+RUN curl -s -L https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key add - && \
+    distribution=$(. /etc/os-release;echo $ID$VERSION_ID) && \
+    curl -s -L https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list && \
+    apt-get update && \
+    apt-get install -y nvidia-container-runtime-hook && \
+    cd /usr && ln -s lib/x86_64-linux-gnu lib64 && cd - && \
+    sed -i 's/^#root/root/; s;@/sbin/ldconfig.real;@/run/nvidia/driver/sbin/ldconfig.real;' /etc/nvidia-container-runtime/config.toml
+
+COPY nvidia-driver /usr/local/bin
+
+WORKDIR /usr/src/nvidia-$DRIVER_VERSION
+
+ARG PUBLIC_KEY=empty
+COPY ${PUBLIC_KEY} kernel/pubkey.x509
+
+# Add NGC DL license
+COPY --from=license /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+ENTRYPOINT ["nvidia-driver"]

README.md

@@ -0,0 +1,98 @@
+# NVIDIA Driver container for CoreOS
+
+## Based on https://gitlab.com/nvidia/container-images/driver/-/tree/main/coreos?ref_type=heads
+
+### Prerequisites
+
+##### Enable required modules
+```sh
+sudo modprobe -a loop ipmi_msghandler
+echo -e "loop\nipmi_msghandler" | sudo tee /etc/modules-load.d/driver.conf
+```
+
+#### Add nvidia-runtime
+```bash
+sudo mkdir -p /etc/systemd/system/docker.service.d
+
+sudo tee /etc/systemd/system/docker.service.d/override.conf <<EOF
+[Service]
+Environment=PATH=$PATH:/run/nvidia/driver/usr/bin
+ExecStart=
+ExecStart=/usr/bin/dockerd --host=fd:// --add-runtime=nvidia=/run/nvidia/driver/usr/bin/nvidia-container-runtime
+EOF
+
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+
+# Make sure the runtime is added
+docker info | grep -i "runtime"
+```
+
+#### CoreOS AWS image
+```sh
+# Run driver container in detached mode
+
+docker run -d --privileged --pid=host --restart=unless-stopped -v /run/nvidia:/run/nvidia:shared --name nvidia-driver nvidia/driver:410.72-4.14.81-coreos
+
+# Check logs to make sure driver container ran properly
+
+docker logs -f nvidia-driver
+
+# Test nvidia-smi with official CUDA image
+
+docker run --runtime=nvidia --rm nvidia/cuda:9.2-base sh -c 'uname -r && nvidia-smi --query-gpu=driver_version --format=csv,noheader'
+```
+
+#### In Kubernetes
+```sh
+# Set up the cluster for Container Linux (https://kubernetes.io/docs/setup/independent/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl)
+
+# Run driver container 
+
+docker run -d --privileged --pid=host --restart=unless-stopped -v /run/nvidia:/run/nvidia:shared --name nvidia-driver nvidia/driver:410.72-4.14.81-coreos
+
+# Make sure the driver container is running before moving on to next steps
+docker ps -a | grep -i driver
+
+# Set default runtime to nvidia by editing /etc/systemd/system/docker.service.d
+
+sudo sed -i "s|nvidia=/run/nvidia/driver/usr/bin/nvidia-container-runtime|nvidia=/run/nvidia/driver/usr/bin/nvidia-container-runtime default-runtime=nvidia|" /etc/systemd/system/docker.service.d/override.conf
+
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+
+# Deploy nvidia k8s-device-plugin (https://github.com/NVIDIA/k8s-device-plugin#enabling-gpu-support-in-kubernetes)
+
+# Deploy GPU pods
+
+# Set up monitoring
+
+# Install helm (https://docs.helm.sh/using_helm/#installing-the-helm-client) to /opt/bin
+
+# Initialize helm
+
+kubectl create serviceaccount tiller --namespace kube-system
+kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller	 
+helm init --service-account tiller
+
+# Label GPU nodes
+
+kubectl label nodes <coreos-gpu-node> hardware-type=NVIDIAGPU
+
+# Install the monitoring charts
+
+helm repo add gpu-helm-charts https://nvidia.github.io/gpu-monitoring-tools/helm-charts
+helm repo update
+helm install gpu-helm-charts/prometheus-operator --name prometheus-operator --namespace monitoring
+helm install gpu-helm-charts/kube-prometheus --name kube-prometheus --namespace monitoring
+
+# Check the status of the pods
+
+kubectl get pods -n monitoring
+
+# Forward the port for Grafana
+
+kubectl -n monitoring port-forward $(kubectl get pods -n monitoring -lapp=kube-prometheus-grafana -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') 3000 &
+
+# Open a browser window and type http://localhost:3000 to view the Nodes Dashboard in Grafana
+```

empty

@@ -0,0 +1,26 @@
+
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+[[constraint]]
+  name = "github.com/opencontainers/runtime-spec"
+  version = "1.0.1"

nvidia-container-runtime/Gopkg.lock

@@ -0,0 +1,16 @@
+RUNTIME=nvidia-container-runtime
+MOCK_RUNC=$(CURDIR)/runc
+
+all:
+	@go build  -o ${RUNTIME}
+
+mock-runc:
+	@(echo '#!/bin/bash\necho mock runc') > ${MOCK_RUNC}
+	@chmod +x ${MOCK_RUNC}
+
+test: all mock-runc
+	@go test -v
+	@${RM} ${MOCK_RUNC}
+
+clean:
+	@${RM} ${RUNTIME}