add nvidia monitor and device plugin crds

Author: devxpy

chart/templates/daemonset.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: "{{ $.Release.Name }}-nvidia-monitor"
+  namespace: "{{ $.Values.namespace }}"
+  labels:
+    app: "{{ $.Release.Name }}-nvidia-monitor"
+spec:
+  selector:
+    matchLabels:
+      name: "{{ $.Release.Name }}-nvidia-monitor"
+  template:
+    metadata:
+      labels:
+        name: "{{ $.Release.Name }}-nvidia-monitor"
+    spec:
+      nodeSelector:
+        gpuEnabled: "true"
+        gpuType: a100
+      tolerations:
+        - effect: NoSchedule
+          key: kubernetes.azure.com/scalesetpriority
+          operator: Equal
+          value: spot
+{{/*      hostNetwork: true*/}}
+      hostPID: true
+      hostIPC: true
+      containers:
+        - name: monitor
+{{/*          securityContext:*/}}
+{{/*            capabilities:*/}}
+{{/*              add:*/}}
+{{/*                - SYS_ADMIN*/}}
+{{/*            privileged: true*/}}
+{{/*            runAsNonRoot: false*/}}
+{{/*            runAsUser: 0*/}}
+          image: "nvcr.io/nvidia/cuda:12.5.0-runtime-ubuntu22.04"
+          volumeMounts:
+            - name: host
+              mountPath: /host
+          args:
+            - sleep
+            - infinity
+      volumes:
+        - name: host
+          hostPath:
+            path: /

nvidia-device-plugin-ds.yaml

@@ -0,0 +1,49 @@
+# this installs the NVIDIA device plugin: https://learn.microsoft.com/en-us/azure/aks/gpu-cluster?tabs=add-ubuntu-gpu-node-pool#manually-install-the-nvidia-device-plugin
+#   kubectl create namespace gpu-resources
+#   kubectl apply -f nvidia-device-plugin-ds.yaml
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-device-plugin-daemonset
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: nvidia-device-plugin-ds
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: nvidia-device-plugin-ds
+    spec:
+      nodeSelector:
+        gpuEnabled: "true"
+      tolerations:
+        - effect: NoSchedule
+          key: kubernetes.azure.com/scalesetpriority
+          operator: Equal
+          value: spot
+      # Mark this pod as a critical add-on; when enabled, the critical add-on
+      # scheduler reserves resources for critical add-on pods so that they can
+      # be rescheduled after a failure.
+      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+      priorityClassName: "system-node-critical"
+      containers:
+        - image: nvcr.io/nvidia/k8s-device-plugin:v0.15.0
+          name: nvidia-device-plugin-ctr
+          env:
+            - name: FAIL_ON_INIT_ERROR
+              value: "false"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ "ALL" ]
+          volumeMounts:
+            - name: device-plugin
+              mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+        - name: device-plugin
+          hostPath:
+            path: /var/lib/kubelet/device-plugins

nvidia-monitor.Dockerfile

@@ -0,0 +1,9 @@
+FROM nvcr.io/nvidia/cuda:12.5.0-runtime-ubuntu22.04
+
+# "https://github.com/containerd/nerdctl/releases/download/v1.7.5/nerdctl-full-1.7.5-linux-amd64.tar.gz" \
+RUN wget -qO nerdctl.tar.gz "https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-1.7.6-linux-amd64.tar.gz" \
+    && tar Cxzvvf /usr/local nerdctl.tar.gz \
+    && rm nerdctl.tar.gz
+
+# for i in $(nerdctl  -a /host/run/containerd/containerd.sock -n k8s.io container ls --format "{{.ID}}"); do nerdctl  -a /host/run/containerd/containerd.sock -n k8s.io inspect -f '{{.State.Pid}} {{index .Config.Labels "io.kubernetes.pod.name"}}' $i; done | grep gooey-gpu
+# nvidia-smi