MIXQ/generate.sh at main · lioZ129/MIXQ · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90


if [ $2 == a100 ]
    then
    CMD=" srun  -N 1 --pty --gres=gpu:a100:1 -p octave -A public python "
fi

if [ $2 == h100 ]
    then
    CMD="srun  -p twills -A h100 --gres=gpu:h100:1 --export=ALL python"
fi

export http_proxy=127.0.0.1:7890
export https_proxy=127.0.0.1:7890
set -x

quantpath=/home/dataset/quant/quant
modelpath=/home/dataset

for batch in   32
#for batch in  1

    do
    for seq in   1024
        do
            ##model_type=Aquila2
            #model_type=opt
            #model_type=Mistral
            #model_type=gpt-j
            #model_type=falcon
            model_type=$3


            data_types=( "mix"  )
            for bit in   8
                do
                for data_type in "${data_types[@]}"
                    do
                    model=${model_type}
                    echo ${model}
                    rm -r ${quantpath}${bit}/${model}/model.safetensors
                    CUDA_VISIBLE_DEVICES=$1    ${CMD}  generate.py  --model_type ${data_type} --model_path  \
                    ${quantpath}${bit}/${model} \
                    --quant_file ${quantpath}${bit}/${model} \
                    --n_batch ${batch} --n_ctx ${batch}  --dataset_path /home/chenyidong/checkpoint/dataset
                done
            done


            # data_types=( "quik"  )
            # for bit in  4
            #     do
            #     for data_type in "${data_types[@]}"
            #         do
            #         model=${model_type}
            #         echo ${model}
            #         CUDA_VISIBLE_DEVICES=$1  ${CMD}  generate.py  --model_type ${data_type} --model_path  \
            #         ${quantpath}quik${bit}/${model} \
            #         --quant_file ${quantpath}quik${bit}/${model} \
            #         --batch_size ${batch} --bit ${bit} --dataset_path /home/chenyidong/checkpoint/dataset
            #     done
            # done
            # data_types=(   "bitsandbytes"   )
            # for data_type in "${data_types[@]}"
            #     do
            #     model=${model_type}

            #     echo ${model}
            #     CUDA_VISIBLE_DEVICES=$1 ${CMD} benchflops.py  --model_type ${data_type} --model_path  \
            #     ${modelpath}/${model} \
            #     --quant_file ${modelpath}/${model} --batch_size ${batch} --dataset_path /home/chenyidong/checkpoint/dataset

            # done
            # data_types=( "awq"   )
            # for data_type in "${data_types[@]}"
            #     do
            #     for model in "${models[@]}"
            #         do
            #         echo ${model}
            #         CUDA_VISIBLE_DEVICES=$1    ${CMD} benchflops.py  --model_type ${data_type} --model_path  \
            #         ${quantpath}/awqquant/${model} \
            #         --quant_file ${quantpath}t/awqquant/${model} --batch_size ${batch}
            #     done
            # done


        done
done