|
12 | 12 | */
|
13 | 13 | public class Qwen3ChatFormat implements ChatFormat {
|
14 | 14 |
|
15 |
| - protected Qwen3Tokenizer tokenizer; |
16 |
| - protected ChatTokens chatTokens; |
17 |
| - |
18 | 15 | protected final int beginOfText;
|
19 | 16 | protected final int startHeader;
|
20 | 17 | protected final int endHeader;
|
21 | 18 | protected final int endOfTurn;
|
22 | 19 | protected final int endOfText;
|
23 | 20 | protected final int endOfMessage;
|
24 | 21 | protected final int endOfTextFim;
|
25 |
| - |
26 | 22 | protected final int imStart; // beginOfText
|
27 | 23 | protected final int imEnd; // endOfText
|
28 |
| - |
29 | 24 | protected final int fimPrefix;
|
30 | 25 | protected final int fimSuffix;
|
31 | 26 | protected final int fimMiddle;
|
| 27 | + protected Qwen3Tokenizer tokenizer; |
| 28 | + protected ChatTokens chatTokens; |
32 | 29 |
|
33 | 30 | public Qwen3ChatFormat(Qwen3Tokenizer tokenizer, ChatTokens chatTokens) {
|
34 | 31 | this.tokenizer = tokenizer;
|
@@ -106,15 +103,12 @@ public int getBeginOfText() {
|
106 | 103 | return beginOfText;
|
107 | 104 | }
|
108 | 105 |
|
109 |
| - |
110 |
| - |
111 | 106 | @Override
|
112 | 107 | public Set<Integer> getStopTokens() {
|
113 | 108 | if (imEnd == -1 && endOfText == -1) {
|
114 | 109 | throw new IllegalStateException("No stop token is defined.");
|
115 | 110 | }
|
116 | 111 | if (imEnd == -1) {
|
117 |
| - |
118 | 112 | return Set.of(endOfText);
|
119 | 113 | }
|
120 | 114 | return Set.of(imEnd, endOfText, endOfTextFim);
|
|
0 commit comments