Skip to content

Commit 12027be

Browse files
committed
revert tests/assets/tokenizer vlm changes
1 parent 3b9860e commit 12027be

File tree

2 files changed

+1
-31
lines changed

2 files changed

+1
-31
lines changed

tests/assets/tokenizer/tokenizer.json

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2029,10 +2029,7 @@
20292029
"land": 1994,
20302030
"?\n": 1995,
20312031
" respect": 1996,
2032-
"ances": 1997,
2033-
"<|image|>": 1998,
2034-
"<|begin_of_image|>": 1999,
2035-
"<|end_of_image|>": 2000
2032+
"ances": 1997
20362033
},
20372034
"merges": [
20382035
]

tests/assets/tokenizer/tokenizer_config.json

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -15,38 +15,11 @@
1515
"rstrip": false,
1616
"single_word": false,
1717
"special": true
18-
},
19-
"1998": {
20-
"content": "<|image|>",
21-
"lstrip": false,
22-
"normalized": false,
23-
"rstrip": false,
24-
"single_word": false,
25-
"special": true
26-
},
27-
"1999": {
28-
"content": "<|begin_of_image|>",
29-
"lstrip": false,
30-
"normalized": false,
31-
"rstrip": false,
32-
"single_word": false,
33-
"special": true
34-
},
35-
"2000": {
36-
"content": "<|end_of_image|>",
37-
"lstrip": false,
38-
"normalized": false,
39-
"rstrip": false,
40-
"single_word": false,
41-
"special": true
4218
}
4319
},
4420
"bos_token": "<|begin_of_text|>",
4521
"clean_up_tokenization_spaces": true,
4622
"eos_token": "<|end_of_text|>",
47-
"img_token": "<|image|>",
48-
"boi_token": "<|begin_of_image|>",
49-
"eoi_token": "<|end_of_image|>",
5023
"model_input_names": [
5124
"input_ids",
5225
"attention_mask"

0 commit comments

Comments
 (0)