SilVar/eval_configs/evaluate.yaml at main · Hanhpt23/SilVar · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
model:
  arch: silvar
  model_type: pretrain
  max_txt_len: 1024
  image_size: 224
  end_sym: </s>
  language_model: meta-llama/Meta-Llama-3-8B
  vision_model: biomed_clip
  audio_model: whisper
  ckpt: path/to/checkpoint
  use_grad_checkpoint: true
  chat_template: true
  low_resource: true
  lora_r: 64
  lora_alpha: 16
  bits: 8
datasets:
  audio_train:
    audio_processor:
      train:
        name: whisper_processor
    vis_processor:
      train:
        name: blip2_image_eval
        image_size: 224
    text_processor:
      train:
        name: blip_caption
    build_info:
      image_path: /train/images
      ann_path: train/train.json
      audio_path: /train/audio_wav
evaluation_datasets:
  audio_val:
    eval_file_path: /test/test.json
    img_path: /test/images
    prompt_test: ''
    audio_path: /test/audio_wav
    batch_size: 2
    max_new_tokens: 512
    temperature: 0.1
    top_p: 0.9
    do_sample: true
run:
  task: image_text_pretrain
  name: evaluation
  save_path: path/to/output/directory