Skip to content

Commit f56dc0b

Browse files
committed
fix:整理代码
1 parent ea4e997 commit f56dc0b

File tree

1 file changed

+9
-3
lines changed

1 file changed

+9
-3
lines changed

src/deepseekTokenizer.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,12 +152,18 @@ class SimpleBPETokenizer {
152152
}
153153
const words = text.split(' ');
154154
for (const word of words) {
155-
if (!word) continue;
155+
if (!word) {
156+
continue;
157+
}
156158
const bpeTokens = this.bpe(word);
157159
tokenCount += bpeTokens.length;
158-
if (tokenCount >= limit) return false; // 超过限制,立即返回
160+
if (tokenCount >= limit) {
161+
return false; // 超过限制,立即返回
162+
}
163+
}
164+
if (this.addEosToken) {
165+
tokenCount++; // EOS token
159166
}
160-
if (this.addEosToken) tokenCount++; // EOS token
161167
return tokenCount < limit;
162168
}
163169
}

0 commit comments

Comments
 (0)