withcatai · giladgd · May 17, 2025 · May 11, 2025 · May 11, 2025 · May 11, 2025
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -3,6 +3,8 @@ description: Report a reproducible bug
 labels:
   - requires triage
   - bug
+title: "bug: "
+type: "Bug"
 body:
   - type: markdown
     attributes:

diff --git a/.github/ISSUE_TEMPLATE/documentation-issue.yml b/.github/ISSUE_TEMPLATE/documentation-issue.yml
@@ -3,6 +3,8 @@ description: Documentation is unclear or otherwise insufficient.
 labels:
   - requires triage
   - documentation
+title: "docs: "
+type: "Documentation"
 body:
   - type: markdown
     attributes:

diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml
@@ -3,6 +3,8 @@ description: Suggest an new idea for this project
 labels:
   - requires triage
   - new feature
+title: "feat: "
+type: "Feature"
 body:
   - type: markdown
     attributes:

diff --git a/.gitignore b/.gitignore
@@ -14,8 +14,10 @@ node_modules
 /.vitepress/.cache
 /test/.models
 /test/temp
+/test/.temp
 /temp
 /coverage
+/test-runner-profile
 
 /llama/compile_commands.json
 /llama/llama.cpp

diff --git a/.vitepress/config.ts b/.vitepress/config.ts
@@ -470,8 +470,6 @@ export default defineConfig({
             }
         },
         sidebar: {
-            "/api/": getApiReferenceSidebar(),
-
             "/guide/": [{
                 text: "Guide",
                 base: "/guide",
@@ -550,7 +548,9 @@ export default defineConfig({
                         ]
                     }
                 ]
-            }]
+            }],
+
+            "/api/": getApiReferenceSidebar()
         },
         socialLinks: [
             {icon: "npm", link: "https://www.npmjs.com/package/node-llama-cpp"},

diff --git a/docs/cli/pull.md b/docs/cli/pull.md
@@ -20,7 +20,7 @@ If a file already exists and its size matches the expected size, it will not be
 
 The supported URI schemes are:
 - **HTTP:** `https://`, `http://`
-- **Hugging Face:** `hf:<user>/<model>:<quant>` (`#<quant>` is optional, [but recommended](../guide/downloading-models.md#hf-scheme-specify-quant))
+- **Hugging Face:** `hf:<user>/<model>:<quant>` (`:<quant>` is optional, [but recommended](../guide/downloading-models.md#hf-scheme-specify-quant))
 - **Hugging Face:** `hf:<user>/<model>/<file-path>#<branch>` (`#<branch>` is optional)
 
 Learn more about using model URIs in the [Downloading Models guide](../guide/downloading-models.md#model-uris).

diff --git a/docs/guide/CUDA.md b/docs/guide/CUDA.md
@@ -114,6 +114,33 @@ set NODE_LLAMA_CPP_CMAKE_OPTION_CMAKE_GENERATOR_TOOLSET=%CUDA_PATH%
 
 Then run the build command again to check whether setting the `CMAKE_GENERATOR_TOOLSET` cmake option fixed the issue.
 
+### Fix the `forward compatibility was attempted on non supported HW` Error {#fix-cuda-forward-compatibility}
+This error usually happens when the CUDA version you have installed on your machine is older than the CUDA version used in the prebuilt binaries supplied by `node-llama-cpp`.
+
+To resolve this issue, you can either [update your CUDA installation](https://developer.nvidia.com/cuda-downloads) to the latest version (recommended) or [build `node-llama-cpp` on your machine](#building) against the CUDA version you have installed.
+
+### Fix the `Binary GPU type mismatch. Expected: cuda, got: false` Error {#fix-cuda-gpu-type-mismatch}
+This error usually happens when you have multiple conflicting CUDA versions installed on your machine.
+
+To fix it, uninstall older CUDA versions and restart your machine (important).
+
+:::: details Check which CUDA libraries are picked up by `node-llama-cpp`'s prebuilt binaries on your machine
+
+Run this command inside of your project:
+
+::: code-group
+```shell [Linux]
+ldd ./node_modules/@node-llama-cpp/linux-x64-cuda/bins/linux-x64-cuda/libggml-cuda.so
+```
+
+```cmd [Windows]
+"C:\Program Files\Git\usr\bin\ldd.exe" node_modules\@node-llama-cpp\win-x64-cuda\bins\win-x64-cuda\ggml-cuda.dll
+```
+:::
+
+::::
+
+
 ## Using `node-llama-cpp` With CUDA
 It's recommended to use [`getLlama`](../api/functions/getLlama) without specifying a GPU type,
 so it'll detect the available GPU types and use the best one automatically.

diff --git a/docs/guide/awesome.md b/docs/guide/awesome.md
@@ -2,17 +2,32 @@
 description: Awesome projects that use node-llama-cpp
 ---
 # Awesome `node-llama-cpp`
-Awesome projects that use `node-llama-cpp`.
+:sunglasses: Awesome projects that use `node-llama-cpp`.
+
+<script setup lang="ts">
+import DataBadge from "../../.vitepress/components/DataBadge/DataBadge.vue";
+</script>
 
 ## Open Source
 * [CatAI](https://github.com/withcatai/catai) - a simplified AI assistant API for Node.js, with REST API support
+  <br /><DataBadge title="License" content="MIT"/>
+
+* [Manzoni](https://manzoni.app/) ([GitHub](https://github.com/gems-platforms/manzoni-app)) - a text editor running local LLMs
+  <br /><DataBadge title="License" content="AGPL-3.0"/>
+
 
 ## Proprietary
-> List your project here!
+* [BashBuddy](https://bashbuddy.run) ([GitHub](https://github.com/wosherco/bashbuddy)) - write bash commands with natural language
+  <br /><DataBadge title="Partially open source" content="Source available" href="https://github.com/wosherco/bashbuddy/blob/main/LICENSE.md"/>
+
+* [nutshell](https://withnutshell.com) - Private AI meeting notes processed completely on your device
+
 
 
 <br />
 
 ---
 
+> To add a project to this list, [open a PR](https://github.com/withcatai/node-llama-cpp/edit/master/docs/guide/awesome.md).
+>
 > To have a project listed here, it should clearly state that it uses `node-llama-cpp`.
diff --git a/docs/guide/chat-session.md b/docs/guide/chat-session.md
@@ -446,6 +446,87 @@ console.log("AI: " + a2);
 ```
 :::
 
+:::: details Saving and restoring a context sequence evaluation state {#save-and-restore-with-context-sequence-state}
+You can also save and restore the context sequence evaluation state to avoid re-evaluating the chat history
+when you load it on a new context sequence.
+
+Please note that context sequence state files can get very large (109MB for only 1K tokens).
+Using this feature is only recommended when the chat history is very long and you plan to load it often,
+or when the evaluation is too slow due to hardware limitations.
+
+::: warning
+When loading a context sequence state from a file,
+always ensure that the model used to create the context sequence is exactly the same as the one used to save the state file.
+
+Loading a state file created from a different model can crash the process,
+thus you have to pass `{acceptRisk: true}` to the [`loadStateFromFile`](../api/classes/LlamaContextSequence.md#loadstatefromfile) method to use it.
+
+Use with caution.
+:::
+
+::: code-group
+```typescript [Save chat history and context sequence state]
+import {fileURLToPath} from "url";
+import path from "path";
+import fs from "fs/promises";
+import {getLlama, LlamaChatSession} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const llama = await getLlama();
+const model = await llama.loadModel({
+    modelPath: path.join(__dirname, "models", "Meta-Llama-3.1-8B-Instruct.Q4_K_M.gguf")
+});
+const context = await model.createContext();
+const contextSequence = context.getSequence();
+const session = new LlamaChatSession({contextSequence});
+
+
+const q1 = "Hi there, how are you?";
+console.log("User: " + q1);
+
+const a1 = await session.prompt(q1);
+console.log("AI: " + a1);
+
+const chatHistory = session.getChatHistory();// [!code highlight]
+await Promise.all([// [!code highlight]
+    contextSequence.saveStateToFile("state.bin"),// [!code highlight]
+    fs.writeFile("chatHistory.json", JSON.stringify(chatHistory), "utf8")// [!code highlight]
+]);// [!code highlight]
+```
+:::
+
+::: code-group
+```typescript [Restore chat history and context sequence state]
+import {fileURLToPath} from "url";
+import path from "path";
+import fs from "fs/promises";
+import {getLlama, LlamaChatSession} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+// ---cut---
+const llama = await getLlama();
+const model = await llama.loadModel({
+    modelPath: path.join(__dirname, "models", "Meta-Llama-3.1-8B-Instruct.Q4_K_M.gguf")
+});
+const context = await model.createContext();
+const contextSequence = context.getSequence();
+const session = new LlamaChatSession({contextSequence});
+
+await contextSequence.loadStateFromFile("state.bin", {acceptRisk: true});// [!code highlight]
+const chatHistory = JSON.parse(await fs.readFile("chatHistory.json", "utf8"));// [!code highlight]
+session.setChatHistory(chatHistory);// [!code highlight]
+
+const q2 = "Summarize what you said";
+console.log("User: " + q2);
+
+const a2 = await session.prompt(q2);
+console.log("AI: " + a2);
+```
+:::
+
+::::
+
 ## Prompt Without Updating Chat History {#prompt-without-updating-chat-history}
 Prompt without saving the prompt to the chat history.
 

diff --git a/docs/guide/cmakeOptions.data.ts b/docs/guide/cmakeOptions.data.ts
@@ -90,6 +90,12 @@ function parseCmakeOptions(cmakeListsTxt: string, optionFilter: ((key: string) =
             }
         } else if (option.defaultValue === "${BUILD_SHARED_LIBS_DEFAULT}")
             option.defaultValue = htmlEscapeWithCodeMarkdown("`OFF` on MinGW, `ON` otherwise");
+        else if (option.defaultValue === "${GGML_CUDA_GRAPHS_DEFAULT}")
+            option.defaultValue = htmlEscapeWithCodeMarkdown("`ON`");
+        else if (option.defaultValue === "${GGML_NATIVE_DEFAULT}")
+            option.defaultValue = htmlEscapeWithCodeMarkdown("`OFF` when building for a different architecture,\n`ON` otherwise");
+        else if (option.key === "LLAMA_CURL")
+            option.defaultValue = htmlEscapeWithCodeMarkdown("`OFF`");
         else
             option.defaultValue = htmlEscapeWithCodeMarkdown(
                 option.defaultValue != null

diff --git a/docs/guide/downloading-models.md b/docs/guide/downloading-models.md
@@ -76,7 +76,7 @@ You can reference models using a URI instead of their full download URL when usi
 When downloading a model from a URI, the model files will be prefixed with a corresponding adaptation of the URI.
 
 To reference a model from Hugging Face, you can use one of these schemes:
-* `hf:<user>/<model>:<quant>` (`#<quant>` is optional, [but recommended](#hf-scheme-specify-quant))
+* `hf:<user>/<model>:<quant>` (`:<quant>` is optional, [but recommended](#hf-scheme-specify-quant))
 * `hf:<user>/<model>/<file-path>#<branch>` (`#<branch>` is optional)
 
 Here are example usages of the Hugging Face URI scheme:

diff --git a/docs/guide/index.md b/docs/guide/index.md
@@ -316,4 +316,5 @@ Explore the [API reference](../api/functions/getLlama.md) to learn more about th
 and use the search bar (press <kbd class="doc-kbd">/</kbd>) to find documentation for a specific topic or API.
 
 Check out the [roadmap](https://github.com/orgs/withcatai/projects/1) to see what's coming next,<br/>
+visit the [awesome list](./awesome.md) to find great projects that use `node-llama-cpp`,<br/>
 and consider [sponsoring `node-llama-cpp`](https://github.com/sponsors/giladgd) to accelerate the development of new features.
diff --git a/docs/guide/low-level-api.md b/docs/guide/low-level-api.md
@@ -391,3 +391,85 @@ console.log(
     newTokens
 );
 ```
+
+### Save and Restore State {#save-and-restore-state}
+You can save the evaluation state of a context sequence to then later load it back.
+
+This is useful for avoiding the evaluation of tokens that you've already evaluated in the past.
+
+::: warning
+When loading a context sequence state from a file,
+always ensure that the model used to create the context sequence is exactly the same as the one used to save the state file.
+
+Loading a state file created from a different model can crash the process,
+thus you have to pass `{acceptRisk: true}` to the [`loadStateFromFile`](../api/classes/LlamaContextSequence.md#loadstatefromfile) method to use it.
+
+Use with caution.
+:::
+
+::: code-group
+```typescript [Save state]
+import {fileURLToPath} from "url";
+import path from "path";
+import {getLlama} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const llama = await getLlama();
+const model = await llama.loadModel({
+    modelPath: path.join(__dirname, "models", "Meta-Llama-3-8B-Instruct.Q4_K_M.gguf")
+});
+const context = await model.createContext();
+const sequence = context.getSequence();
+
+const input = "The best way to";
+const tokens = model.tokenize(input);
+await sequence.evaluateWithoutGeneratingNewTokens(tokens);
+
+console.log(
+    "Current state:",
+    model.detokenize(sequence.contextTokens, true),
+    sequence.contextTokens
+);
+
+await sequence.saveStateToFile("state.bin");// [!code highlight]
+```
+:::
+
+::: code-group
+```typescript [Load state]
+import {fileURLToPath} from "url";
+import path from "path";
+import {getLlama, Token} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+// ---cut---
+const llama = await getLlama();
+const model = await llama.loadModel({
+    modelPath: path.join(__dirname, "models", "Meta-Llama-3-8B-Instruct.Q4_K_M.gguf")
+});
+const context = await model.createContext();
+const sequence = context.getSequence();
+
+await sequence.loadStateFromFile("state.bin", {acceptRisk: true});// [!code highlight]
+
+console.log(
+    "Loaded state:",
+    model.detokenize(sequence.contextTokens, true),
+    sequence.contextTokens
+);
+
+const input = " find";
+const inputTokens = model.tokenize(input);
+const maxTokens = 10;
+const res: Token[] = [];
+for await (const token of sequence.evaluate(inputTokens)) {
+    res.push(token);
+
+    if (res.length >= maxTokens)
+        break;
+}
+
+console.log("Result:", model.detokenize(res));
+```
+:::
diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt
@@ -1,9 +1,17 @@
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.19)
 
 if (NLC_CURRENT_PLATFORM STREQUAL "win-x64" OR NLC_CURRENT_PLATFORM STREQUAL "win-arm64")
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif()
 
+if (NLC_CURRENT_PLATFORM STREQUAL "win-x64")
+    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+        set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebugDLL" CACHE STRING "" FORCE)
+    else()
+        set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDLL" CACHE STRING "" FORCE)
+    endif()
+endif()
+
 if (NLC_TARGET_PLATFORM STREQUAL "win-arm64" AND (CMAKE_GENERATOR STREQUAL "Ninja" OR CMAKE_GENERATOR STREQUAL "Ninja Multi-Config") AND NOT MINGW)
     if(NLC_CURRENT_PLATFORM STREQUAL "win-x64")
         include("./profiles/llvm.win32.host-x64.target-arm64.cmake")
@@ -70,6 +78,9 @@ add_subdirectory("llama.cpp")
 include_directories("llama.cpp")
 include_directories("./llama.cpp/common")
 
+# This is needed to use methods in "llama-grammar.h" and "unicode.h"
+target_include_directories(llama PUBLIC "./llama.cpp/src")
+
 unset(GPU_INFO_HEADERS)
 unset(GPU_INFO_SOURCES)
 unset(GPU_INFO_EXTRA_LIBS)