feat: extract function calling syntax from a Jinja template

docs/guide/electron.md

@@ -9,7 +9,7 @@ Trying to use `node-llama-cpp` on a renderer process will crash the application.
 
 You can scaffold an example Electron app that uses `node-llama-cpp` with complete configuration for packaging and distribution by running the following command:
 ```shell
-npm create node-llama-cpp@latest --template electron-typescript-react
+npm create node-llama-cpp@latest -- --template electron-typescript-react
 ```
 
 ::: tip

docs/guide/function-calling.md

@@ -408,3 +408,42 @@ getFruitPrice({name: "banana"}) result: {name: "banana", price: "$4"};
 
 
 ```
+
+## Troubleshooting {#troubleshooting}
+### Function Calling Issues With [`JinjaTemplateChatWrapper`](../api/classes/JinjaTemplateChatWrapper.md) {#troubleshoot-jinja-function-calling-issues}
+If function calling doesn't work well (or at all) with a model you're trying to use,
+and the [chat wrapper](./chat-wrapper.md) used by your [`LlamaChatSession`](../api/classes/LlamaChatSession.md)
+is a [`JinjaTemplateChatWrapper`](../api/classes/JinjaTemplateChatWrapper.md)
+(you can check that by accessing [`.chatWrapper`](../api/classes/LlamaChatSession.md#chatwrapper)),
+you can try to force it to not use the function calling template defined in the Jinja template.
+
+Doing this can help you achieve better function calling performance with some models.
+
+To do this, create your [`LlamaChatSession`](../api/classes/LlamaChatSession.md) like this:
+```typescript
+import {fileURLToPath} from "url";
+import path from "path";
+import {getLlama} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const llama = await getLlama();
+const model = await llama.loadModel({
+    modelPath: path.join(__dirname, "models", "Meta-Llama-3-8B-Instruct.Q4_K_M.gguf")
+});
+const context = await model.createContext();
+
+// ---cut---
+import {LlamaChatSession, resolveChatWrapper} from "node-llama-cpp";
+
+const session = new LlamaChatSession({
+    contextSequence: context.getSequence(),
+    chatWrapper: resolveChatWrapper(model, {
+        customWrapperSettings: {
+            jinjaTemplate: {
+                functionCallMessageTemplate: "noJinja"
+            }
+        }
+    })
+});
+```

docs/public/giscus/style.css

@@ -31,6 +31,10 @@ body, #__next {
         border-start-start-radius: 0;
     }
 
+    .gsc-comment .gsc-replies {
+        padding-top: 0;
+    }
+
     .gsc-reactions-popover {
         border-radius: 12px;
 

llama/addon/AddonContext.cpp

@@ -583,7 +583,7 @@ Napi::Value AddonContext::DisposeSequence(const Napi::CallbackInfo& info) {
 
     int32_t sequenceId = info[0].As<Napi::Number>().Int32Value();
 
-    bool result = llama_kv_cache_seq_rm(ctx, sequenceId, -1, -1);
+    bool result = llama_kv_self_seq_rm(ctx, sequenceId, -1, -1);
 
     if (!result) {
         Napi::Error::New(info.Env(), "Failed to dispose sequence").ThrowAsJavaScriptException();
@@ -602,7 +602,7 @@ Napi::Value AddonContext::RemoveTokenCellsFromSequence(const Napi::CallbackInfo&
     int32_t startPos = info[1].As<Napi::Number>().Int32Value();
     int32_t endPos = info[2].As<Napi::Number>().Int32Value();
 
-    bool result = llama_kv_cache_seq_rm(ctx, sequenceId, startPos, endPos);
+    bool result = llama_kv_self_seq_rm(ctx, sequenceId, startPos, endPos);
 
     return Napi::Boolean::New(info.Env(), result);
 }
@@ -617,7 +617,7 @@ Napi::Value AddonContext::ShiftSequenceTokenCells(const Napi::CallbackInfo& info
     int32_t endPos = info[2].As<Napi::Number>().Int32Value();
     int32_t shiftDelta = info[3].As<Napi::Number>().Int32Value();
 
-    llama_kv_cache_seq_add(ctx, sequenceId, startPos, endPos, shiftDelta);
+    llama_kv_self_seq_add(ctx, sequenceId, startPos, endPos, shiftDelta);
 
     return info.Env().Undefined();
 }

package.json

@@ -111,6 +111,8 @@
     "catai",
     "mistral",
     "deepseek",
+    "qwen",
+    "qwq",
     "typescript",
     "lora",
     "batching",

src/ChatWrapper.ts

@@ -8,6 +8,7 @@ import {ChatModelFunctionsDocumentationGenerator} from "./chatWrappers/utils/Cha
 import {jsonDumps} from "./chatWrappers/utils/jsonDumps.js";
 import {defaultChatSystemPrompt} from "./config.js";
 import {getChatWrapperSegmentDefinition} from "./utils/getChatWrapperSegmentDefinition.js";
+import type {JinjaTemplateChatWrapperOptions} from "./chatWrappers/generic/JinjaTemplateChatWrapper.js";
 
 export abstract class ChatWrapper {
     public static defaultSettings: ChatWrapperSettings = {
@@ -17,13 +18,15 @@ export abstract class ChatWrapper {
                 optionalPrefixSpace: true,
                 prefix: "||call: ",
                 paramsPrefix: LlamaText(new SpecialTokensText("(")),
-                suffix: LlamaText(new SpecialTokensText(")"))
+                suffix: LlamaText(new SpecialTokensText(")")),
+                emptyCallParamsPlaceholder: ""
             },
             result: {
                 prefix: LlamaText(new SpecialTokensText("\n"), "||result: "),
                 suffix: LlamaText(new SpecialTokensText("\n"))
             }
-        }
+        },
+        segments: {}
     };
 
     public abstract readonly wrapperName: string;
@@ -102,13 +105,16 @@ export abstract class ChatWrapper {
     }
 
     public generateFunctionCall(name: string, params: any): LlamaText {
+        const emptyCallParamsPlaceholder = this.settings.functions.call.emptyCallParamsPlaceholder;
         return LlamaText([
             this.settings.functions.call.prefix,
             name,
             this.settings.functions.call.paramsPrefix,
             (
                 params === undefined
-                    ? ""
+                    ? (emptyCallParamsPlaceholder === undefined || emptyCallParamsPlaceholder === "")
+                        ? ""
+                        : jsonDumps(emptyCallParamsPlaceholder)
                     : jsonDumps(params)
             ),
             this.settings.functions.call.suffix
@@ -139,7 +145,7 @@ export abstract class ChatWrapper {
         ]);
     }
 
-    public generateModelResponseText(modelResponse: ChatModelResponse["response"], useRawCall: boolean = true): LlamaText {
+    public generateModelResponseText(modelResponse: ChatModelResponse["response"], useRawValues: boolean = true): LlamaText {
         const res: LlamaText[] = [];
         const pendingFunctionCalls: ChatModelFunctionCall[] = [];
         const segmentStack: ChatModelSegmentType[] = [];
@@ -150,7 +156,7 @@ export abstract class ChatWrapper {
             if (pendingFunctionCalls.length === 0)
                 return;
 
-            res.push(this.generateFunctionCallsAndResults(pendingFunctionCalls, useRawCall));
+            res.push(this.generateFunctionCallsAndResults(pendingFunctionCalls, useRawValues));
             pendingFunctionCalls.length = 0;
             needsToAddSegmentReminder = true;
         };
@@ -180,11 +186,10 @@ export abstract class ChatWrapper {
             } else if (isChatModelResponseSegment(response)) {
                 addFunctionCalls();
 
-                if (response.raw != null && useRawCall)
+                const segmentDefinition = getChatWrapperSegmentDefinition(this.settings, response.segmentType);
+                if (response.raw != null && useRawValues)
                     res.push(LlamaText.fromJSON(response.raw));
-                else {
-                    const segmentDefinition = getChatWrapperSegmentDefinition(this.settings, response.segmentType);
-
+                else
                     res.push(
                         LlamaText([
                             (segmentStack.length > 0 && segmentStack.at(-1) === response.segmentType)
@@ -197,12 +202,15 @@ export abstract class ChatWrapper {
                         ])
                     );
 
-                    lastSegmentEndedWithoutSuffix = response.ended && segmentDefinition?.suffix == null;
+                lastSegmentEndedWithoutSuffix = response.ended && segmentDefinition?.suffix == null;
+
+                if (!response.ended && segmentStack.at(-1) !== response.segmentType)
+                    segmentStack.push(response.segmentType);
+                else if (response.ended && segmentStack.at(-1) === response.segmentType) {
+                    segmentStack.pop();
 
-                    if (!response.ended)
-                        segmentStack.push(response.segmentType);
-                    else if (segmentStack.at(-1) === response.segmentType)
-                        segmentStack.pop();
+                    if (segmentStack.length === 0 && segmentDefinition?.suffix == null && this.settings.segments?.closeAllSegments != null)
+                        res.push(LlamaText(this.settings.segments.closeAllSegments));
                 }
 
                 continue;
@@ -277,9 +285,7 @@ export abstract class ChatWrapper {
     }
 
     /** @internal */
-    public static _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): (
-        Array<Record<string | symbol, any> | [testConfig: Record<string | symbol, any>, applyConfig: Record<string | symbol, any>]>
-    ) {
+    public static _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [{}] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
     }
 
@@ -293,13 +299,9 @@ type FirstItemOfTupleOrFallback<T extends any[], Fallback> = T extends [infer U,
 
 export type ChatWrapperJinjaMatchConfiguration<T extends typeof ChatWrapper> = Array<
     FirstItemOfTupleOrFallback<ConstructorParameters<T>, object> |
-    [
-        testConfig: FirstItemOfTupleOrFallback<ConstructorParameters<T>, object>,
-        applyConfig: FirstItemOfTupleOrFallback<ConstructorParameters<T>, object>
-    ] |
     [
         testConfig: FirstItemOfTupleOrFallback<ConstructorParameters<T>, object>,
         applyConfig: FirstItemOfTupleOrFallback<ConstructorParameters<T>, object>,
-        testJinjaParameters: Record<string, any>
+        testJinjaChatWrapperOptions?: JinjaTemplateChatWrapperOptions
     ]
 >;

src/chatWrappers/AlpacaChatWrapper.ts

@@ -35,10 +35,10 @@ export class AlpacaChatWrapper extends GeneralChatWrapper {
     }
 
     /** @internal */
-    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [
             {},
             {allowSpecialTokensInTitles: true}
-        ] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
+        ];
     }
 }

src/chatWrappers/DeepSeekChatWrapper.ts

@@ -348,14 +348,14 @@ export class DeepSeekChatWrapper extends ChatWrapper {
     }
 
     /** @internal */
-    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [
-            {},
-            {keepOnlyLastThought: true},
-            {functionCallingSyntax: "simplified"},
-            {functionCallingSyntax: "simplified", keepOnlyLastThought: true},
-            {functionCallingSyntax: "original"},
-            {functionCallingSyntax: "original", keepOnlyLastThought: true}
-        ] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
+            [undefined, {}, {functionCallMessageTemplate: "noJinja"}],
+            [undefined, {keepOnlyLastThought: true}, {functionCallMessageTemplate: "noJinja"}],
+            [undefined, {functionCallingSyntax: "simplified"}, {functionCallMessageTemplate: "noJinja"}],
+            [undefined, {functionCallingSyntax: "simplified", keepOnlyLastThought: true}, {functionCallMessageTemplate: "noJinja"}],
+            [undefined, {functionCallingSyntax: "original"}, {functionCallMessageTemplate: "noJinja"}],
+            [undefined, {functionCallingSyntax: "original", keepOnlyLastThought: true}, {functionCallMessageTemplate: "noJinja"}]
+        ];
     }
 }

src/chatWrappers/FalconChatWrapper.ts

@@ -154,10 +154,10 @@ export class FalconChatWrapper extends ChatWrapper {
     }
 
     /** @internal */
-    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [
             {},
             {allowSpecialTokensInTitles: true}
-        ] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
+        ];
     }
 }

src/chatWrappers/FunctionaryChatWrapper.ts

@@ -204,7 +204,7 @@ export class FunctionaryChatWrapper extends ChatWrapper {
                                         : LlamaText([
                                             new SpecialTokensText(">>>all\n"),
                                             response,
-                                            (isLastItem && isLastResponse)
+                                            (!isLastResponse || isLastItem)
                                                 ? LlamaText([])
                                                 : new SpecialToken("EOT")
                                         ])
@@ -726,12 +726,12 @@ export class FunctionaryChatWrapper extends ChatWrapper {
     }
 
     /** @internal */
-    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [
             {variation: "v3"},
             {variation: "v2.llama3"},
             {variation: "v2"}
-        ] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
+        ];
     }
 }
 

src/chatWrappers/GeneralChatWrapper.ts

@@ -173,10 +173,10 @@ export class GeneralChatWrapper extends ChatWrapper {
     }
 
     /** @internal */
-    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [
             {},
             {allowSpecialTokensInTitles: true}
-        ] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
+        ];
     }
 }

src/chatWrappers/Llama2ChatWrapper.ts

@@ -115,10 +115,10 @@ export class Llama2ChatWrapper extends ChatWrapper {
     }
 
     /** @internal */
-    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [
             {addSpaceBeforeEos: false},
             {addSpaceBeforeEos: true}
-        ] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
+        ];
     }
 }

src/chatWrappers/Llama3_1ChatWrapper.ts

@@ -341,15 +341,19 @@ export class Llama3_1ChatWrapper extends ChatWrapper {
     }
 
     /** @internal */
-    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate() {
+    public static override _getOptionConfigurationsToTestIfCanSupersedeJinjaTemplate(): ChatWrapperJinjaMatchConfiguration<typeof this> {
         return [
-            {},
-            [{todayDate: null}, {}],
-            [{cuttingKnowledgeDate: null}, {}],
-            [{noToolInstructions: true}, {}],
-            [{todayDate: null, cuttingKnowledgeDate: null}, {}],
-            [{todayDate: null, cuttingKnowledgeDate: null, noToolInstructions: true}, {}],
-            [{todayDate: new Date("2024-07-26T00:00:00"), cuttingKnowledgeDate: null, noToolInstructions: true}, {}],
+            [{}, undefined, {functionCallMessageTemplate: "noJinja"}],
+            [{todayDate: null}, {}, {functionCallMessageTemplate: "noJinja"}],
+            [{cuttingKnowledgeDate: null}, {}, {functionCallMessageTemplate: "noJinja"}],
+            [{noToolInstructions: true}, {}, {functionCallMessageTemplate: "noJinja"}],
+            [{todayDate: null, cuttingKnowledgeDate: null}, {}, {functionCallMessageTemplate: "noJinja"}],
+            [{todayDate: null, cuttingKnowledgeDate: null, noToolInstructions: true}, {}, {functionCallMessageTemplate: "noJinja"}],
+            [
+                {todayDate: new Date("2024-07-26T00:00:00"), cuttingKnowledgeDate: null, noToolInstructions: true},
+                {},
+                {functionCallMessageTemplate: "noJinja"}
+            ],
 
             [
                 {
@@ -358,7 +362,10 @@ export class Llama3_1ChatWrapper extends ChatWrapper {
                     noToolInstructions: true
                 },
                 {cuttingKnowledgeDate: new Date("2023-12-01T00:00:00Z")},
-                {"date_string": formatDate(new Date("2024-07-26T00:00:00"), undefined)}
+                {
+                    additionalRenderParameters: {"date_string": formatDate(new Date("2024-07-26T00:00:00"), undefined)},
+                    functionCallMessageTemplate: "noJinja"
+                }
             ],
 
             [
@@ -369,9 +376,12 @@ export class Llama3_1ChatWrapper extends ChatWrapper {
                     _specialTokensTextForPreamble: true
                 },
                 {cuttingKnowledgeDate: new Date("2023-12-01T00:00:00Z")},
-                {"date_string": formatDate(new Date("2024-07-26T00:00:00"), undefined)}
+                {
+                    additionalRenderParameters: {"date_string": formatDate(new Date("2024-07-26T00:00:00"), undefined)},
+                    functionCallMessageTemplate: "noJinja"
+                }
             ]
-        ] satisfies ChatWrapperJinjaMatchConfiguration<typeof this>;
+        ];
     }
 }