Skip to content
This repository was archived by the owner on Sep 10, 2025. It is now read-only.

Commit eab209f

Browse files
Jack-KhuuJack-Khuu
committed
Add benchmarking scripts
1 parent 3ce1cef commit eab209f

File tree

2 files changed

+328
-0
lines changed

2 files changed

+328
-0
lines changed

‎scripts/benchmarking/benchmarking_linux.sh‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
2+
RUN_CUDA_EAGER=true
3+
RUN_CUDA_COMPILE=false
4+
RUN_CUDA_AOTI=false
5+
6+
RUN_CPU_EAGER=true
7+
RUN_CPU_COMPILE=false
8+
RUN_CPU_AOTI=false
9+
10+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
11+
# Check and Set Up Args (model, out_directory)
12+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
13+
if [ $# -ne 2 ]; then
14+
echo "Please provide (1) model and (2) directory as positional arguments"
15+
exit 1
16+
fi
17+
18+
model=$1
19+
dir=$2
20+
21+
mkdir -p $dir
22+
23+
24+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
25+
# Helpers
26+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
27+
28+
# Function for printing and writing to files
29+
function formatted_export_and_generate {
30+
local file="$dir/$1"
31+
local generate_cmd="$2"
32+
local compile_cmd="$3"
33+
34+
# Write Commands to the top of the output file
35+
echo $compile_cmd > $file
36+
echo $generate_cmd >> $file
37+
38+
echo "Writing to: ${file}"
39+
40+
# Export the Model
41+
if [ ! -z "$compile_cmd" ]; then
42+
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" >> $file
43+
echo "$compile_cmd" | tee -a $file
44+
eval $compile_cmd &>> $file
45+
fi
46+
47+
# Generate using the Model
48+
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" >> $file
49+
echo $generate_cmd | tee -a $file
50+
eval $generate_cmd &>> $file
51+
echo
52+
}
53+
54+
55+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
56+
# Cuda eager
57+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
58+
59+
if [ "$RUN_CUDA_EAGER" = "true" ]; then
60+
echo "Cuda eager b16"
61+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --prompt \"Once upon a time,\" --max-new-tokens 200 --num-samples 3"
62+
file="cuda_eager_b16.txt"
63+
formatted_export_and_generate "$file" "$generate_cmd"
64+
65+
echo "Cuda eager int8"
66+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --prompt \"Once upon a time,\" --max-new-tokens 200 --num-samples 3"
67+
file="cuda_eager_8.txt"
68+
formatted_export_and_generate "$file" "$generate_cmd"
69+
70+
echo "Cuda eager int4"
71+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --prompt \"Once upon a time,\" --max-new-tokens 200 --num-samples 3"
72+
file="cuda_eager_4.txt"
73+
formatted_export_and_generate "$file" "$generate_cmd"
74+
fi
75+
76+
77+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
78+
# Cuda compile
79+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
80+
81+
if [ "$RUN_CUDA_COMPILE" = "true" ]; then
82+
echo "Cuda compile b16"
83+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --prompt \"Once upon a time,\" --max-new-tokens 200 --compile --num-samples 3"
84+
file="cuda_compile_b16.txt"
85+
formatted_export_and_generate "$file" "$generate_cmd"
86+
87+
echo "Cuda compile int8"
88+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --prompt \"Once upon a time,\" --max-new-tokens 200 --compile --num-samples 3"
89+
file="cuda_compile_8.txt"
90+
formatted_export_and_generate "$file" "$generate_cmd"
91+
92+
echo "Cuda compile int4"
93+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --prompt \"Once upon a time,\" --max-new-tokens 200 --compile --num-samples 3"
94+
file="cuda_compile_4.txt"
95+
formatted_export_and_generate "$file" "$generate_cmd"
96+
fi
97+
98+
99+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
100+
# CPU eager
101+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
102+
103+
if [ "$RUN_CPU_EAGER" = "true" ]; then
104+
echo "CPU eager b16"
105+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
106+
file="cpu_eager_b16.txt"
107+
formatted_export_and_generate "$file" "$generate_cmd"
108+
109+
echo "CPU eager int8"
110+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
111+
file="cpu_eager_8.txt"
112+
formatted_export_and_generate "$file" "$generate_cmd"
113+
114+
echo "CPU eager int4"
115+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
116+
file="cpu_eager_4.txt"
117+
formatted_export_and_generate "$file" "$generate_cmd"
118+
fi
119+
120+
121+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
122+
# CPU compile
123+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
124+
125+
if [ "$RUN_CPU_COMPILE" = "true" ]; then
126+
echo "CPU compile b16"
127+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --compile --num-samples 3"
128+
file="cpu_compile_b16.txt"
129+
formatted_export_and_generate "$file" "$generate_cmd"
130+
131+
echo "CPU compile int8"
132+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --compile --num-samples 3"
133+
file="cpu_compile_8.txt"
134+
formatted_export_and_generate "$file" "$generate_cmd"
135+
136+
echo "CPU compile int4"
137+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --compile --num-samples 3"
138+
file="cpu_compile_4.txt"
139+
formatted_export_and_generate "$file" "$generate_cmd"
140+
fi
141+
142+
143+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
144+
# Cuda AOTI
145+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
146+
147+
if [ "$RUN_CUDA_AOTI" = "true" ]; then
148+
echo "Cuda aoti b16"
149+
compile_cmd="python3 torchchat.py export $model --quantize '{\"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --output-dso-path /tmp/model16.so"
150+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model16.so --prompt \"Once upon a time,\" --max-new-tokens 200 --device cuda --num-samples 3"
151+
file="cuda_aoti_b16.txt"
152+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
153+
154+
echo "Cuda aoti int8"
155+
compile_cmd="python3 torchchat.py export $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --output-dso-path /tmp/model8.so"
156+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model8.so --prompt \"Once upon a time,\" --max-new-tokens 200 --device cuda --num-samples 3"
157+
file="cuda_aoti_8.txt"
158+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
159+
160+
echo "Cuda aoti int4"
161+
compile_cmd="python3 torchchat.py export $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cuda\"}}' --output-dso-path /tmp/model34.so"
162+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model34.so --prompt \"Once upon a time,\" --max-new-tokens 200 --device cuda --num-samples 3"
163+
file="cuda_aoti_4.txt"
164+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
165+
fi
166+
167+
168+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
169+
# CPU AOTI
170+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
171+
172+
if [ "$RUN_CPU_AOTI" = "true" ]; then
173+
echo "CPU aoti b16"
174+
compile_cmd="python3 torchchat.py export $model --quantize '{\"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --output-dso-path /tmp/model16.so"
175+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model16.so --prompt \"Once upon a time,\" --max-new-tokens 256 --device cpu --num-samples 3"
176+
file="cpu_aoti_b16.txt"
177+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
178+
179+
echo "CPU aoti int8"
180+
compile_cmd="python3 torchchat.py export $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --output-dso-path /tmp/model8.so"
181+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model8.so --prompt \"Once upon a time,\" --max-new-tokens 256 --device cpu --num-samples 3"
182+
file="cpu_aoti_8.txt"
183+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
184+
185+
echo "CPU aoti int4"
186+
compile_cmd="python3 torchchat.py export $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"bfloat16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --output-dso-path /tmp/model34.so"
187+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model34.so --prompt \"Once upon a time,\" --max-new-tokens 256 --device cpu --num-samples 3"
188+
file="cpu_aoti_4.txt"
189+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
190+
fi

‎scripts/benchmarking/benchmarking_mac.sh‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
2+
RUN_MPS_EAGER=false
3+
4+
RUN_CPU_EAGER=true
5+
RUN_CPU_COMPILE=false
6+
RUN_CPU_AOTI=false
7+
8+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
9+
# Check and Set Up Args (model, out_directory)
10+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
11+
if [ $# -ne 2 ]; then
12+
echo "Please provide (1) model and (2) directory as positional arguments"
13+
exit 1
14+
fi
15+
16+
model=$1
17+
dir=$2
18+
19+
mkdir -p $dir
20+
21+
22+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
23+
# Helpers
24+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
25+
26+
# Function for printing and writing to files
27+
function formatted_export_and_generate {
28+
local file="$dir/$1"
29+
local generate_cmd="$2"
30+
local compile_cmd="$3"
31+
32+
# Write Commands to the top of the output file
33+
echo $compile_cmd > $file
34+
echo $generate_cmd >> $file
35+
36+
echo "Writing to: ${file}"
37+
38+
# Export the Model
39+
if [ ! -z "$compile_cmd" ]; then
40+
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" >> $file
41+
echo "$compile_cmd" | tee -a $file
42+
eval $compile_cmd >> $file 2>&1
43+
fi
44+
45+
# Generate using the Model
46+
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" >> $file
47+
echo $generate_cmd | tee -a $file
48+
eval $generate_cmd >> $file 2>&1
49+
echo
50+
}
51+
52+
53+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
54+
# MPS Eager
55+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
56+
57+
if [ "$RUN_MPS_EAGER" = "true" ]; then
58+
echo "MPS Eager 16"
59+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"mps\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
60+
file="mps_eager_16.txt"
61+
formatted_export_and_generate "$file" "$generate_cmd"
62+
63+
echo "MPS Eager int8"
64+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"mps\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
65+
file="mps_eager_8.txt"
66+
formatted_export_and_generate "$file" "$generate_cmd"
67+
68+
echo "MPS Eager int4"
69+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"mps\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
70+
file="mps_eager_4.txt"
71+
formatted_export_and_generate "$file" "$generate_cmd"
72+
fi
73+
74+
75+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
76+
# CPU Eager
77+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
78+
79+
if [ "$RUN_CPU_EAGER" = "true" ]; then
80+
echo "CPU Eager 16"
81+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
82+
file="cpu_eager_16.txt"
83+
formatted_export_and_generate "$file" "$generate_cmd"
84+
85+
echo "CPU Eager int8"
86+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
87+
file="cpu_eager_8.txt"
88+
formatted_export_and_generate "$file" "$generate_cmd"
89+
90+
echo "CPU Eager int4"
91+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --num-samples 3"
92+
file="cpu_eager_4.txt"
93+
formatted_export_and_generate "$file" "$generate_cmd"
94+
fi
95+
96+
97+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
98+
# CPU compile
99+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
100+
if [ "$RUN_CPU_COMPILE" = "true" ]; then
101+
echo "CPU compile b16"
102+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --compile --num-samples 3"
103+
file="cpu_compile_b16.txt"
104+
formatted_export_and_generate "$file" "$generate_cmd"
105+
106+
echo "CPU compile int8"
107+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --compile --num-samples 3"
108+
file="cpu_compile_8.txt"
109+
formatted_export_and_generate "$file" "$generate_cmd"
110+
111+
echo "CPU compile int4"
112+
generate_cmd="python3 torchchat.py generate $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --prompt \"Once upon a time,\" --max-new-tokens 256 --compile --num-samples 3"
113+
file="cpu_compile_4.txt"
114+
formatted_export_and_generate "$file" "$generate_cmd"
115+
fi
116+
117+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
118+
# CPU AOTI
119+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
120+
if [ "$RUN_CPU_AOTI" = "true" ]; then
121+
echo "CPU aoti b16"
122+
compile_cmd="python3 torchchat.py export $model --quantize '{\"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --output-dso-path /tmp/model16.so"
123+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model16.so --prompt \"Once upon a time,\" --max-new-tokens 256 --device cpu --num-samples 3"
124+
file="cpu_aoti_b16.txt"
125+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
126+
127+
echo "CPU aoti int8"
128+
compile_cmd="python3 torchchat.py export $model --quantize '{\"linear:int8\": {\"groupsize\": 0}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --output-dso-path /tmp/model8.so"
129+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model8.so --prompt \"Once upon a time,\" --max-new-tokens 256 --device cpu --num-samples 3"
130+
file="cpu_aoti_8.txt"
131+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
132+
133+
echo "CPU aoti int4"
134+
compile_cmd="python3 torchchat.py export $model --quantize '{\"linear:int4\": {\"groupsize\": 256}, \"precision\": {\"dtype\":\"float16\"}, \"executor\":{\"accelerator\":\"cpu\"}}' --output-dso-path /tmp/model34.so"
135+
generate_cmd="python3 torchchat.py generate $model --dso-path /tmp/model34.so --prompt \"Once upon a time,\" --max-new-tokens 256 --device cpu --num-samples 3"
136+
file="cpu_aoti_4.txt"
137+
formatted_export_and_generate "$file" "$generate_cmd" "$compile_cmd"
138+
fi

0 commit comments

Comments
 (0)