Collect GPU usage metrics with prometheus

helm-charts/support/Chart.yaml

@@ -57,4 +57,11 @@ dependencies:
   - name: jupyterhub-cost-monitoring
     version: "v1.1.1"
     repository: https://2i2c.org/jupyterhub-cost-monitoring/
-    condition: jupyterhub-cost-monitoring.enabled
+    condition: jupyterhub-cost-monitoring.enabled
+
+  # Provide metrics about GPU usage
+  # https://github.com/NVIDIA/dcgm-exporter
+  - name: dcgm-exporter
+    version: 4.8.1
+    repository: https://nvidia.github.io/dcgm-exporter/helm-charts
+    condition: dcgm-exporter.enabled

helm-charts/support/values.schema.yaml

@@ -42,6 +42,9 @@ properties:
   global:
     type: object
     additionalProperties: true
+  dcgm-exporter:
+    type: object
+    additionalProperties: true
 
   # These provide values for objects we create, so we validate their schema
   # to the best of our ability.

helm-charts/support/values.yaml

@@ -461,6 +461,21 @@ jupyterhub-cost-monitoring:
     cpu: 5m
     memory: 100Mi
 
+dcgm-exporter:
+  enabled: false
+  serviceMonitor:
+    enabled: false
+  podAnnotations:
+    prometheus.io/path: "/metrics"
+    prometheus.io/port: "12121"
+    prometheus.io/scrape: "true"
+  tolerations:
+  - key: nvidia.com/gpu
+    operator: Equal
+    value: present
+    effect: NoSchedule
+
+
 # Configuration of templates provided directly by this chart
 # -------------------------------------------------------------------------------
 #