|
13 | 13 |
|
14 | 14 | from QEfficient import QEFFAutoModelForImageTextToText |
15 | 15 |
|
16 | | -## For AWQ model update pytorch version to 2.8.* |
| 16 | +# For AWQ model update pytorch version to 2.8.* |
17 | 17 | model_id = "Qwen/Qwen2.5-VL-3B-Instruct" |
18 | 18 | config = AutoConfig.from_pretrained(model_id) |
19 | 19 | config.text_config.num_hidden_layers = 2 |
|
24 | 24 | tokenizer = transformers.AutoTokenizer.from_pretrained(model_id) |
25 | 25 | processor = AutoProcessor.from_pretrained(model_id) |
26 | 26 |
|
27 | | -### use skip_vision=Ture, if want to run only text, ow false ### |
| 27 | +# use skip_vision=True, if want to run only text |
28 | 28 | skip_vision = False |
29 | 29 |
|
30 | | -if skip_vision: |
31 | | - ## Only Text ## |
32 | | - |
33 | | - ## Set Batch_Size ## |
| 30 | +if skip_vision: # Only Text |
34 | 31 | batch_size = 1 |
35 | 32 | qeff_model.compile( |
36 | 33 | batch_size=batch_size, |
|
77 | 74 | print(tokenizer.batch_decode(output.generated_ids)) |
78 | 75 | print(output) |
79 | 76 |
|
80 | | -else: |
| 77 | +else: # Vision + Text |
81 | 78 | batch_size = 1 |
82 | 79 | ctx_len = 14336 |
83 | 80 | widths = [360, 320, 360, 454, 536, 640, 720, 910, 720, 1280, 1920] |
84 | 81 | heights = [120, 180, 240, 256, 354, 360, 480, 512, 576, 720, 1080] |
85 | 82 | num_frames = [177, 139, 78, 64, 37, 30, 20, 16, 16, 7, 7] |
86 | 83 |
|
87 | | - ## Vision + Text ## |
88 | 84 | qeff_model.compile( |
89 | 85 | batch_size=batch_size, |
90 | 86 | prefill_seq_len=128, |
|
104 | 100 | mos=1, |
105 | 101 | ) |
106 | 102 |
|
107 | | - ### IMAGE + TEXT ### |
108 | 103 | image_url = "https://picsum.photos/id/237/536/354" |
109 | 104 | image = Image.open(requests.get(image_url, stream=True).raw) |
110 | | - image = image.resize((360, 120)) # Resize to any deimnsion present in specializations (width, height) |
| 105 | + image = image.resize((360, 120)) # Resize to any dimension (width, height) present in specializations |
111 | 106 | messages_1 = [ |
112 | 107 | { |
113 | 108 | "role": "user", |
|
0 commit comments