feat: parallel function calling (#225)

* feat: parallel function calling
* feat: preload prompt
* feat: prompt completion engine
* feat: chat wrapper based system message support
* feat: add prompt completion to the Electron example
* feat: model compatibility warnings
* feat: Functionary `v2.llama3` support
* feat: parallel function calling with plain Llama 3 Instruct
* feat: improve function calling support for default chat wrapper
* feat: parallel model downloads
* feat: add the electron example to releases
* feat: improve the electron example
* feat: `customStopTriggers` for `LlamaCompletion`
* fix: improve CUDA detection on Windows
* fix: performance improvements
* refactor: make `functionCallMessageTemplate` an object
* chore: adapt to `llama.cpp` breaking changes

Author: giladgd

.config/typedoc.json

@@ -24,5 +24,6 @@
     "enumMembersFormat": "table",
     "typeDeclarationFormat": "list",
     "sort": ["source-order"],
-    "docsRoot": "../docs"
+    "docsRoot": "../docs",
+    "intentionallyNotExported": ["MergeOptionalUnionTypes", "GbnfJsonSchemaToTSType", "_LlamaText"]
 }

.github/workflows/build.yml

@@ -67,7 +67,7 @@ jobs:
             generators: "Ninja"
             artifact: "linux"
           - name: "macOS Clang"
-            os: macos-12
+            os: macos-13
             cc: "clang"
             cxx: "clang++"
             generators: "Xcode"
@@ -110,18 +110,18 @@ jobs:
 
       - name: Install Cuda on Windows
         if: startsWith(matrix.config.os, 'windows')
-        uses: Jimver/[email protected].11
+        uses: Jimver/[email protected].15
         with:
-          cuda: '12.2.0'
+          cuda: '12.4.1'
           method: 'network'
           sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
           use-local-cache: false
 
       - name: Install Cuda on Ubuntu
         if: startsWith(matrix.config.name, 'Ubuntu GCC')
-        uses: Jimver/[email protected].11
+        uses: Jimver/[email protected].15
         with:
-          cuda: '12.2.0'
+          cuda: '12.4.1'
           method: 'network'
 
       - name: Install Vulkan SDK on Windows
@@ -198,7 +198,7 @@ jobs:
           async function buildBinary(arch, flags = [], nodeTarget = nodeVersion) {
             console.log(`Building ${arch} for node ${nodeTarget} with flags`, flags);
             
-            await $`node ./dist/cli/cli.js build --noUsageExample --arch ${arch} --nodeTarget ${nodeVersion} ${flags}`;
+            await $`node ./dist/cli/cli.js build --ciMode --noUsageExample --arch ${arch} --nodeTarget ${nodeVersion} ${flags}`;
           }
           
           // build binaries
@@ -343,6 +343,8 @@ jobs:
     needs:
       - build
       - build-binaries
+    outputs:
+      package-version: ${{ steps.set-package-version.outputs.package-version }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -391,6 +393,12 @@ jobs:
           if [ -f .semanticRelease.npmPackage.deployedVersion.txt ]; then
             echo "npm-url=https://www.npmjs.com/package/node-llama-cpp/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT
           fi
+      - name: Set package version to GITHUB_OUTPUT
+        id: set-package-version
+        run: |
+          if [ -f .semanticRelease.npmPackage.deployedVersion.txt ]; then
+            echo "package-version=$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT
+          fi
       - name: Prepare `create-node-llama-cpp` module
         if: steps.set-npm-url.outputs.npm-url != ''
         run: |
@@ -437,3 +445,56 @@ jobs:
         uses: actions/deploy-pages@v4
         with:
           artifact_name: pages-docs
+
+  build-electron-example:
+    name: Build & release Electron app example - ${{ matrix.config.name }}
+    needs:
+      - release
+    if: needs.release.outputs.package-version != ''
+    runs-on: ${{ matrix.config.os }}
+    permissions:
+      contents: write
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - name: "Windows"
+            os: windows-2022
+          - name: "Ubuntu"
+            os: ubuntu-22.04
+          - name: "macOS"
+            os: macos-13
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install modules
+        run: npm ci
+
+      - name: Create Electron app project
+        env:
+          DEPLOYED_PACKAGE_VERSION: ${{ needs.release.outputs.package-version }}
+        run: |
+          npx --no vite-node ./scripts/scaffoldElectronExampleForCiBuild.ts --packageVersion "$DEPLOYED_PACKAGE_VERSION" --packageFolderPath ./electron-app-example
+          cd electron-app-example
+          npm install
+
+      - name: Build electron app
+        id: build
+        shell: bash
+        timeout-minutes: 480
+        run: |
+          cd electron-app-example
+          npm run build
+          ls ./release
+
+      - name: Add builds to current release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file: "electron-app-example/release/*.{dmg,zip,exe,appx,AppImage,snap,deb,tar.gz}"
+          file_glob: true
+          tag: ${{ needs.release.outputs.package-version }}
+          make_latest: false

llama/addon.cpp

@@ -222,9 +222,9 @@ static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* mod
         return Napi::Number::From(info.Env(), -1);
     }
 
-    auto tokenType = llama_token_get_type(model, token);
+    auto tokenAttributes = llama_token_get_attr(model, token);
 
-    if (tokenType == LLAMA_TOKEN_TYPE_UNDEFINED || tokenType == LLAMA_TOKEN_TYPE_UNKNOWN) {
+    if (tokenAttributes & LLAMA_TOKEN_ATTR_UNDEFINED || tokenAttributes & LLAMA_TOKEN_ATTR_UNKNOWN) {
         return Napi::Number::From(info.Env(), -1);
     }
 
@@ -236,9 +236,9 @@ static Napi::Value getNapiControlToken(const Napi::CallbackInfo& info, llama_mod
         return Napi::Number::From(info.Env(), -1);
     }
 
-    auto tokenType = llama_token_get_type(model, token);
+    auto tokenAttributes = llama_token_get_attr(model, token);
 
-    if (tokenType != LLAMA_TOKEN_TYPE_CONTROL && tokenType != LLAMA_TOKEN_TYPE_USER_DEFINED) {
+    if (!(tokenAttributes & LLAMA_TOKEN_ATTR_CONTROL) && !(tokenAttributes & LLAMA_TOKEN_ATTR_UNDEFINED)) {
         return Napi::Number::From(info.Env(), -1);
     }
 
@@ -535,20 +535,20 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
             return Napi::String::New(info.Env(), ss.str());
         }
 
-        Napi::Value GetTokenType(const Napi::CallbackInfo& info) {
+        Napi::Value GetTokenAttributes(const Napi::CallbackInfo& info) {
             if (disposed) {
                 Napi::Error::New(info.Env(), "Model is disposed").ThrowAsJavaScriptException();
                 return info.Env().Undefined();
             }
 
             if (info[0].IsNumber() == false) {
-                return Napi::Number::From(info.Env(), int32_t(LLAMA_TOKEN_TYPE_UNDEFINED));
+                return Napi::Number::From(info.Env(), int32_t(LLAMA_TOKEN_ATTR_UNDEFINED));
             }
 
             int token = info[0].As<Napi::Number>().Int32Value();
-            auto tokenType = llama_token_get_type(model, token);
+            auto tokenAttributes = llama_token_get_attr(model, token);
 
-            return Napi::Number::From(info.Env(), int32_t(tokenType));
+            return Napi::Number::From(info.Env(), int32_t(tokenAttributes));
         }
         Napi::Value IsEogToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -611,7 +611,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                         InstanceMethod("suffixToken", &AddonModel::SuffixToken),
                         InstanceMethod("eotToken", &AddonModel::EotToken),
                         InstanceMethod("getTokenString", &AddonModel::GetTokenString),
-                        InstanceMethod("getTokenType", &AddonModel::GetTokenType),
+                        InstanceMethod("getTokenAttributes", &AddonModel::GetTokenAttributes),
                         InstanceMethod("isEogToken", &AddonModel::IsEogToken),
                         InstanceMethod("getVocabularyType", &AddonModel::GetVocabularyType),
                         InstanceMethod("shouldPrependBosToken", &AddonModel::ShouldPrependBosToken),