Skip to content

Commit 6ae52b0

Browse files
giladgdgiladgd
committed
fix: adapt Llama chat wrappers to breaking llama.cpp changes
1 parent adc3fcb commit 6ae52b0

File tree

9 files changed

+85
-128
lines changed

9 files changed

+85
-128
lines changed

‎src/chatWrappers/FunctionaryChatWrapper.ts‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,13 @@ export class FunctionaryChatWrapper extends ChatWrapper {
3939
prefix: LlamaText([
4040
new SpecialTokensText("<|start_header_id|>tool<|end_header_id|>\n\n")
4141
]),
42-
suffix: LlamaText(new SpecialToken("EOT"))
42+
suffix: LlamaText(new SpecialTokensText("<|eot_id|>"))
4343
},
4444
parallelism: {
4545
call: {
4646
sectionPrefix: "",
4747
betweenCalls: "",
48-
sectionSuffix: LlamaText(new SpecialToken("EOT"))
48+
sectionSuffix: LlamaText(new SpecialTokensText("<|eot_id|>"))
4949
},
5050
result: {
5151
sectionPrefix: "",
@@ -72,13 +72,13 @@ export class FunctionaryChatWrapper extends ChatWrapper {
7272
"{{functionName}}",
7373
new SpecialTokensText("\n")
7474
]),
75-
suffix: LlamaText(new SpecialToken("EOT"))
75+
suffix: LlamaText(new SpecialTokensText("<|eot_id|>"))
7676
},
7777
parallelism: {
7878
call: {
7979
sectionPrefix: "",
8080
betweenCalls: "",
81-
sectionSuffix: LlamaText(new SpecialToken("EOT"))
81+
sectionSuffix: LlamaText(new SpecialTokensText("<|eot_id|>"))
8282
},
8383
result: {
8484
sectionPrefix: "",
@@ -155,13 +155,13 @@ export class FunctionaryChatWrapper extends ChatWrapper {
155155
return LlamaText([
156156
new SpecialTokensText("<|start_header_id|>system<|end_header_id|>\n\n"),
157157
LlamaText.fromJSON(item.text),
158-
new SpecialToken("EOT")
158+
new SpecialTokensText("<|eot_id|>")
159159
]);
160160
} else if (item.type === "user") {
161161
return LlamaText([
162162
new SpecialTokensText("<|start_header_id|>user<|end_header_id|>\n\n"),
163163
item.text,
164-
new SpecialToken("EOT")
164+
new SpecialTokensText("<|eot_id|>")
165165
]);
166166
} else if (item.type === "model") {
167167
if (isLastItem && item.response.length === 0)
@@ -178,7 +178,7 @@ export class FunctionaryChatWrapper extends ChatWrapper {
178178
return;
179179

180180
res.push(LlamaText(pendingFunctionCalls));
181-
res.push(LlamaText(new SpecialToken("EOT")));
181+
res.push(LlamaText(new SpecialTokensText("<|eot_id|>")));
182182
res.push(LlamaText(pendingFunctionResults));
183183

184184
pendingFunctionResults.length = 0;
@@ -206,7 +206,7 @@ export class FunctionaryChatWrapper extends ChatWrapper {
206206
response,
207207
(!isLastResponse || isLastItem)
208208
? LlamaText([])
209-
: new SpecialToken("EOT")
209+
: new SpecialTokensText("<|eot_id|>")
210210
])
211211
])
212212
);
@@ -232,7 +232,7 @@ export class FunctionaryChatWrapper extends ChatWrapper {
232232
response.result === undefined
233233
? "" // "void"
234234
: jsonDumps(response.result),
235-
new SpecialToken("EOT")
235+
new SpecialTokensText("<|eot_id|>")
236236
])
237237
);
238238
} else
@@ -320,13 +320,13 @@ export class FunctionaryChatWrapper extends ChatWrapper {
320320
return LlamaText([
321321
new SpecialTokensText("<|start_header_id|>system<|end_header_id|>\n\n"),
322322
LlamaText.fromJSON(item.text),
323-
new SpecialToken("EOT")
323+
new SpecialTokensText("<|eot_id|>")
324324
]);
325325
} else if (item.type === "user") {
326326
return LlamaText([
327327
new SpecialTokensText("<|start_header_id|>user<|end_header_id|>\n\n"),
328328
item.text,
329-
new SpecialToken("EOT")
329+
new SpecialTokensText("<|eot_id|>")
330330
]);
331331
} else if (item.type === "model") {
332332
if (isLastItem && item.response.length === 0)
@@ -343,7 +343,7 @@ export class FunctionaryChatWrapper extends ChatWrapper {
343343
return;
344344

345345
res.push(LlamaText(pendingFunctionCalls));
346-
res.push(LlamaText(new SpecialToken("EOT")));
346+
res.push(LlamaText(new SpecialTokensText("<|eot_id|>")));
347347
res.push(LlamaText(pendingFunctionResults));
348348

349349
pendingFunctionResults.length = 0;
@@ -365,7 +365,7 @@ export class FunctionaryChatWrapper extends ChatWrapper {
365365
response,
366366
(isLastItem && isLastResponse)
367367
? LlamaText([])
368-
: new SpecialToken("EOT")
368+
: new SpecialTokensText("<|eot_id|>")
369369
])
370370
);
371371
} else if (isChatModelResponseFunctionCall(response)) {
@@ -392,7 +392,7 @@ export class FunctionaryChatWrapper extends ChatWrapper {
392392
response.result === undefined
393393
? "" // "void"
394394
: jsonDumps(response.result),
395-
new SpecialToken("EOT")
395+
new SpecialTokensText("<|eot_id|>")
396396
])
397397
);
398398
} else

‎src/chatWrappers/Llama3ChatWrapper.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,13 @@ export class Llama3ChatWrapper extends ChatWrapper {
3434
},
3535
result: {
3636
prefix: LlamaText(new SpecialTokensText("<|start_header_id|>function_call_result<|end_header_id|>\n\n")),
37-
suffix: LlamaText(new SpecialToken("EOT"))
37+
suffix: LlamaText(new SpecialTokensText("<|eot_id|>"))
3838
},
3939
parallelism: {
4040
call: {
4141
sectionPrefix: "",
4242
betweenCalls: "\n",
43-
sectionSuffix: LlamaText(new SpecialToken("EOT"))
43+
sectionSuffix: LlamaText(new SpecialTokensText("<|eot_id|>"))
4444
},
4545
result: {
4646
sectionPrefix: "",
@@ -62,11 +62,11 @@ export class Llama3ChatWrapper extends ChatWrapper {
6262
},
6363
result: {
6464
prefix: LlamaText([
65-
LlamaText(new SpecialToken("EOT")),
65+
LlamaText(new SpecialTokensText("<|eot_id|>")),
6666
new SpecialTokensText("<|start_header_id|>function_call_result<|end_header_id|>\n\n")
6767
]),
6868
suffix: LlamaText([
69-
new SpecialToken("EOT"),
69+
new SpecialTokensText("<|eot_id|>"),
7070
new SpecialTokensText("<|start_header_id|>assistant<|end_header_id|>\n\n")
7171
])
7272
}
@@ -147,7 +147,7 @@ export class Llama3ChatWrapper extends ChatWrapper {
147147
LlamaText([
148148
new SpecialTokensText("<|start_header_id|>system<|end_header_id|>\n\n"),
149149
item.system,
150-
new SpecialToken("EOT")
150+
new SpecialTokensText("<|eot_id|>")
151151
])
152152
);
153153
}
@@ -157,7 +157,7 @@ export class Llama3ChatWrapper extends ChatWrapper {
157157
LlamaText([
158158
new SpecialTokensText("<|start_header_id|>user<|end_header_id|>\n\n"),
159159
item.user,
160-
new SpecialToken("EOT")
160+
new SpecialTokensText("<|eot_id|>")
161161
])
162162
);
163163
}
@@ -169,7 +169,7 @@ export class Llama3ChatWrapper extends ChatWrapper {
169169
item.model,
170170
isLastItem
171171
? LlamaText([])
172-
: new SpecialToken("EOT")
172+
: new SpecialTokensText("<|eot_id|>")
173173
])
174174
);
175175
}

‎src/chatWrappers/Llama3_1ChatWrapper.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export class Llama3_1ChatWrapper extends ChatWrapper {
2929
},
3030
result: {
3131
prefix: LlamaText(new SpecialTokensText("\n<|start_header_id|>ipython<|end_header_id|>\n\n")),
32-
suffix: LlamaText(new SpecialToken("EOT"), new SpecialTokensText("<|start_header_id|>assistant<|end_header_id|>\n\n"))
32+
suffix: LlamaText(new SpecialTokensText("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"))
3333
}
3434
}
3535
};
@@ -189,7 +189,7 @@ export class Llama3_1ChatWrapper extends ChatWrapper {
189189
LlamaText([
190190
new SpecialTokensText("<|start_header_id|>system<|end_header_id|>\n\n"),
191191
item.system,
192-
new SpecialToken("EOT")
192+
new SpecialTokensText("<|eot_id|>")
193193
])
194194
);
195195
}
@@ -199,7 +199,7 @@ export class Llama3_1ChatWrapper extends ChatWrapper {
199199
LlamaText([
200200
new SpecialTokensText("<|start_header_id|>user<|end_header_id|>\n\n"),
201201
item.user,
202-
new SpecialToken("EOT")
202+
new SpecialTokensText("<|eot_id|>")
203203
])
204204
);
205205
}
@@ -211,7 +211,7 @@ export class Llama3_1ChatWrapper extends ChatWrapper {
211211
item.model,
212212
isLastItem
213213
? LlamaText([])
214-
: new SpecialToken("EOT")
214+
: new SpecialTokensText("<|eot_id|>")
215215
])
216216
);
217217
}

‎src/chatWrappers/Llama3_2LightweightChatWrapper.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,12 @@ export class Llama3_2LightweightChatWrapper extends ChatWrapper {
2424
optionalPrefixSpace: true,
2525
prefix: '{"name": "',
2626
paramsPrefix: '", "parameters": ',
27-
suffix: LlamaText("}", new SpecialToken("EOT")),
27+
suffix: LlamaText("}", new SpecialTokensText("<|eot_id|>")),
2828
emptyCallParamsPlaceholder: {}
2929
},
3030
result: {
31-
prefix: LlamaText(new SpecialToken("EOT"), new SpecialTokensText("<|start_header_id|>ipython<|end_header_id|>\n\n")),
32-
suffix: LlamaText(new SpecialToken("EOT"), new SpecialTokensText("<|start_header_id|>assistant<|end_header_id|>\n\n"))
31+
prefix: LlamaText(new SpecialTokensText("<|eot_id|><|start_header_id|>ipython<|end_header_id|>\n\n")),
32+
suffix: LlamaText(new SpecialTokensText("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"))
3333
}
3434
}
3535
};
@@ -192,7 +192,7 @@ export class Llama3_2LightweightChatWrapper extends ChatWrapper {
192192
LlamaText([
193193
new SpecialTokensText("<|start_header_id|>system<|end_header_id|>\n\n"),
194194
item.system,
195-
new SpecialToken("EOT")
195+
new SpecialTokensText("<|eot_id|>")
196196
])
197197
);
198198
}
@@ -202,7 +202,7 @@ export class Llama3_2LightweightChatWrapper extends ChatWrapper {
202202
LlamaText([
203203
new SpecialTokensText("<|start_header_id|>user<|end_header_id|>\n\n"),
204204
item.user,
205-
new SpecialToken("EOT")
205+
new SpecialTokensText("<|eot_id|>")
206206
])
207207
);
208208
}
@@ -214,7 +214,7 @@ export class Llama3_2LightweightChatWrapper extends ChatWrapper {
214214
item.model,
215215
isLastItem
216216
? LlamaText([])
217-
: new SpecialToken("EOT")
217+
: new SpecialTokensText("<|eot_id|>")
218218
])
219219
);
220220
}

‎test/modelDependent/llama3.2/promptCompletion.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ describe("llama 3.2", () => {
5858
5959
You are a helpful, respectful and honest assistant. Always answer as helpfully as possible.
6060
If a question does not make any sense, or is not factually coherent, explain why instead of answering something incorrectly. If you don't know the answer to a question, don't share false information.",
61-
new SpecialToken("EOT"),
61+
new SpecialToken("EOS"),
6262
new SpecialTokensText("<|start_header_id|>"),
6363
"user",
6464
new SpecialTokensText("<|end_header_id|>"),
@@ -86,14 +86,14 @@ describe("llama 3.2", () => {
8686
8787
You are a helpful, respectful and honest assistant. Always answer as helpfully as possible.
8888
If a question does not make any sense, or is not factually coherent, explain why instead of answering something incorrectly. If you don't know the answer to a question, don't share false information.",
89-
new SpecialToken("EOT"),
89+
new SpecialToken("EOS"),
9090
new SpecialTokensText("<|start_header_id|>"),
9191
"user",
9292
new SpecialTokensText("<|end_header_id|>"),
9393
"
9494
9595
Hi there!",
96-
new SpecialToken("EOT"),
96+
new SpecialToken("EOS"),
9797
new SpecialTokensText("<|start_header_id|>"),
9898
"assistant",
9999
new SpecialTokensText("<|end_header_id|>"),

0 commit comments

Comments
 (0)