cluster setup

.dockerignore

@@ -0,0 +1,4 @@
+examples
+data
+temp
+wandb

Dockerfile

@@ -0,0 +1,5 @@
+FROM registry.ferrari.snucse.org:30443/attention-x/drive-vision-assistant:latest
+COPY . /root
+WORKDIR /root
+RUN apt-get update && apt-get -y install libgl1-mesa-glx ffmpeg libsm6 libxext6
+RUN pip install -r requirements.txt

deploy-and-experiment.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+docker build -t registry.ferrari.snucse.org:30443/attention-x/drive-vision-assistant-experiment:latest .
+docker push registry.ferrari.snucse.org:30443/attention-x/drive-vision-assistant-experiment:latest
+kubectl delete pod vision-assistant-experiment
+kubectl create -f ./resources/pod/experiment.yaml

experiment.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+kubectl delete pod vision-assistant-experiment
+kubectl create -f ./resources/pod/experiment.yaml

resources/pod/experiment.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vision-assistant-experiment
+spec:
+  restartPolicy: Never
+
+  containers:
+  - name: experiment-container
+    image: "registry.ferrari.snucse.org:30443/attention-x/drive-vision-assistant-experiment:latest"
+    command: ["torchrun"]
+    args: [ "--nproc_per_node=1", "train.py", "--cfg-path", "train_configs/drive_finetune.yaml" ]
+    imagePullPolicy: Always
+    resources:
+      limits:
+        cpu: "50"
+        memory: "400Gi"
+        nvidia.com/gpu: "4"
+    envFrom:
+      - secretRef:
+            name: wandb-secret
+    volumeMounts:
+    - mountPath: "/data"
+      name: vision-assistant-pv
+  volumes:
+  - name: vision-assistant-pv
+    persistentVolumeClaim:
+      claimName: vision-assistant-pvc

resources/secrets/wandb.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: wandb-secret
+type: Opaque
+stringData:
+  WANDB_API_KEY: "" # wandb api key
+  LC_ALL: "C.UTF-8"
+  LANG: "C.UTF-8"
+  LANGUAGE: "C.UTF-8"