VideoActionModel/scripts/evaluate_ego_trajectory.slurm at main · valeoai/VideoActionModel · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
#!/bin/bash
#SBATCH --job-name=ego_eval
#SBATCH -A ycy@h100
#SBATCH -C h100
#SBATCH --nodes=4
#SBATCH --ntasks-per-node=4
#SBATCH --cpus-per-task=24
#SBATCH --gres=gpu:4
#SBATCH --hint=nomultithread
#SBATCH --qos=qos_gpu_h100-gc
#SBATCH --time=10:10:00
#SBATCH --output=slurm_jobs_logs/stdout/ego_eval/%A_%a.out
#SBATCH --error=slurm_jobs_logs/stdout/ego_eval/%A_%a.out
#SBATCH --array=0-8

mkdir -p slurm_jobs_logs/stdout/ego_eval
mkdir -p slurm_jobs_logs/files/ego_eval


module purge
module load arch/h100
module load pytorch-gpu/py3/2.4.0
export PYTHONUSERBASE=$WORK/python_envs/video_action_model
export MPICH_GPU_SUPPORT_ENABLED=1
export CUDA_LAUNCH_BLOCKING=1
export HYDRA_FULL_ERROR=1
# Important change when using deepspeed (which now uses triton)
# By default the cache dir will be $HOME/.triton
# We point it to $SCRATCH because the inodes quota is very limited on JeanZay
export TRITON_CACHE_DIR=$SCRATCH/.triton
export TMPDIR=$JOBSCRATCH

# Echo des commandes lancees
set -x

ckpt[0]=./tmp/vam_ckpt/action_expert_38k_attndim768_actdim192.ckpt
batch_size[0]=64
outdir[0]=action_expert_38k_attndim768_actdim192_5samples

ckpt[1]=./tmp/vam_ckpt/action_expert_77k_attndim768_actdim192.ckpt
batch_size[1]=64
outdir[1]=action_expert_77k_attndim768_actdim192_5samples

ckpt[2]=./tmp/vam_ckpt/action_expert_116k_attndim768_actdim192.ckpt
batch_size[2]=64
outdir[2]=action_expert_116k_attndim768_actdim192_5samples

ckpt[3]=./tmp/vam_ckpt/action_expert_139k_attndim768_actdim192.ckpt
batch_size[3]=64
outdir[3]=action_expert_139k_attndim768_actdim192_5samples

ckpt[4]=./tmp/vam_ckpt/action_expert_38k_attndim1024_actdim256.ckpt
batch_size[4]=64
outdir[4]=action_expert_38k_attndim1024_actdim256_5samples

ckpt[5]=./tmp/vam_ckpt/action_expert_77k_attndim1024_actdim256.ckpt
batch_size[5]=64
outdir[5]=action_expert_77k_attndim1024_actdim256_5samples

ckpt[6]=./tmp/vam_ckpt/action_expert_116k_attndim1024_actdim256.ckpt
batch_size[6]=64
outdir[6]=action_expert_116k_attndim1024_actdim256_5samples

ckpt[7]=./tmp/vam_ckpt/action_expert_139k_attndim1024_actdim256.ckpt
batch_size[7]=64
outdir[7]=action_expert_139k_attndim1024_actdim256_5samples

ckpt[8]=./tmp/vam_ckpt/action_expert_139k_attndim2048_actdim512.ckpt
batch_size[8]=32
outdir[8]=action_expert_139k_attndim2048_actdim512_5samples


srun python scripts/evaluate_ego_trajectory.py \
--vam_checkpoint_path ${ckpt[${SLURM_ARRAY_TASK_ID}]} \
--outdir slurm_jobs_logs/files/ego_eval/${outdir[${SLURM_ARRAY_TASK_ID}]} \
--num_workers 24 \
--batch_size ${batch_size[${SLURM_ARRAY_TASK_ID}]} \
--store_trajectories True \
--num_sampled_trajectories 5 \
--dtype bf16