Skip to content

Commit 97abbca

Browse files
giladgdgiladgd
authored
feat: Llama 3.2 3B function calling support (#386)
* feat: Llama 3.2 3B function calling support * feat: use `llama.cpp` backend registry for GPUs instead of custom implementations * feat(`getLlama`): `build: "try"` option * feat(`init` command): `--model` flag * fix: improve binary compatibility testing on Electron apps * fix: too many abort signal listeners * fix: log level of some lower level logs * fix: context window missing response during generation on specific extreme conditions * fix: adapt to breaking `llama.cpp` changes * fix: automatically resolve `compiler is out of heap space` CUDA build error * chore: update bug report template * docs: separate open source and proprietary sections in the awesome list * docs(troubleshooting): Electron build error on Windows * docs(Electron): GitHub Actions template for cross-compilation
1 parent 6405ee9 commit 97abbca

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1890
-633
lines changed

‎.github/ISSUE_TEMPLATE/bug-report.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@ body:
2727
attributes:
2828
label: Actual Behavior
2929
description: >-
30-
A clear and concise description of what actually happened. Please wrap
31-
any error messages or output in code tags, instead of images.
30+
A clear and concise description of what actually happened.
31+
Please wrap any error messages or output in code tags, instead of images.
32+
Please attach logs if relevant.
3233
validations:
3334
required: true
3435
- type: textarea
@@ -60,6 +61,11 @@ body:
6061
| Node.js version | x.y.zzz |
6162
| Typescript version | x.y.zzz |
6263
| `node-llama-cpp` version | x.y.zzz |
64+
65+
`npx --yes node-llama-cpp inspect gpu` output:
66+
```
67+
Result of running `npx --yes node-llama-cpp inspect gpu`
68+
```
6369
validations:
6470
required: true
6571
- type: textarea

‎.github/workflows/build.yml‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,12 @@ jobs:
6565
- name: "Ubuntu"
6666
os: ubuntu-22.04
6767
artifact: "linux"
68-
- name: "macOS"
68+
- name: "macOS x64"
6969
os: macos-13
70-
artifact: "mac"
70+
artifact: "mac-x64"
71+
- name: "macOS arm64"
72+
os: macos-14
73+
artifact: "mac-arm64"
7174

7275
steps:
7376
- uses: actions/checkout@v4
@@ -87,11 +90,18 @@ jobs:
8790
name: llama.cpp
8891
path: llama
8992

90-
- name: Install dependencies on Windows
91-
if: startsWith(matrix.config.os, 'windows')
93+
- name: Install dependencies on Windows for x64
94+
if: matrix.config.name == 'Windows for x64'
9295
run: |
9396
choco install ninja cmake
9497
98+
- name: Install dependencies on Windows
99+
if: matrix.config.name == 'Windows for Arm'
100+
run: |
101+
choco install cmake.install --version=3.31.1
102+
choco install cmake --version=3.31.1
103+
choco install ninja
104+
95105
- name: Install dependencies on Ubuntu
96106
if: matrix.config.name == 'Ubuntu'
97107
run: |
@@ -148,7 +158,7 @@ jobs:
148158
- name: Setup & Build
149159
id: build
150160
shell: bash
151-
timeout-minutes: 200
161+
timeout-minutes: 300
152162
env:
153163
ARTIFACT_NAME: ${{ matrix.config.artifact }}
154164
run: |
@@ -212,9 +222,10 @@ jobs:
212222
await buildBinary("x64", ["--gpu", "vulkan"]);
213223
await buildBinary("arm64", ["--gpu", "false"]);
214224
await buildBinary("armv7l", ["--gpu", "false"]);
215-
} else if (process.env.ARTIFACT_NAME === "mac") {
216-
await buildBinary("arm64", ["--gpu", "metal"]);
225+
} else if (process.env.ARTIFACT_NAME === "mac-x64") {
217226
await buildBinary("x64", ["--gpu", "false"]);
227+
} else if (process.env.ARTIFACT_NAME === "mac-arm64") {
228+
await buildBinary("arm64", ["--gpu", "metal"]);
218229
}
219230
220231
// move binaries to bins

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ node_modules
2424
/llama/lastBuild.json
2525
/llama/gitRelease.bundle
2626
/llama/.temp
27+
/llama/.cache
28+
/llama/build
2729
/llama/.idea
30+
/llama/.vscode
2831
/llama/cmake-build-debug
2932
/llama/localBuilds
3033
/llama/Release

‎.vitepress/assets/ogTemplate.svg‎

Lines changed: 5 additions & 5 deletions
Loading

‎.vitepress/config.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,12 +122,22 @@ export default defineConfig({
122122

123123
return items
124124
.map((item) => {
125-
if (item.url === "" || item.url === "blog/") {
125+
if (item.url === "") {
126+
item.lastmod = undefined;
127+
item.changefreq = "daily";
128+
item.priority = 1;
129+
} else if (item.url === "blog/") {
126130
item.lastmod = new Date(buildDate);
131+
item.changefreq = "daily";
132+
item.priority = 0.9;
127133
} else if (item.url.startsWith("api/") || item.url.startsWith("cli/")) {
128134
item = {
129135
...item,
130-
lastmod: new Date(buildDate)
136+
lastmod: new Date(buildDate),
137+
changefreq: "weekly",
138+
priority: item.url.startsWith("cli/")
139+
? 0.7
140+
: 0.5
131141
};
132142
} else if (item.lastmod == null && item.url.startsWith("blog/")) {
133143
const postDate = blogPostMap.get(item.url)?.frontmatter.date;
@@ -138,6 +148,13 @@ export default defineConfig({
138148
}
139149
} else if (item.lastmod == null) {
140150
item.lastmod = new Date(buildDate);
151+
item.changefreq = "weekly";
152+
item.priority = 0.4;
153+
}
154+
155+
if (item.url !== "blog/" && item.url.startsWith("blog/")) {
156+
item.priority = 0.8;
157+
item.changefreq = "hourly";
141158
}
142159

143160
return item;

‎docs/guide/awesome.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
# Awesome `node-llama-cpp`
22
Awesome projects that use `node-llama-cpp`.
33

4-
---
5-
4+
## Open Source
65
* [CatAI](https://github.com/withcatai/catai) - a simplified AI assistant API for Node.js, with REST API support
76

7+
## Proprietary
8+
> List your project here!
9+
10+
11+
<br />
12+
813
---
914

1015
> To have a project listed here, it should clearly state that it uses `node-llama-cpp`.

‎docs/guide/electron.md‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,67 @@ You also need to call [`getLlama`](../api/functions/getLlama.md) with the CMake
3434
so that `node-llama-cpp` can find them.
3535

3636
## Cross Compilation
37-
Cross packaging from one platform to another is not supported, since binaries for other platforms are not downloaded to you machine when your run `npm install`.
37+
Cross packaging from one platform to another is not supported, since binaries for other platforms are not downloaded to your machine when you run `npm install`.
3838

3939
Packaging an `arm64` app on an `x64` machine is supported, but packaging an `x64` app on an `arm64` machine is not.
4040

41+
::: details GitHub Actions template for cross-compilation
42+
43+
<span v-pre>
44+
45+
```yml
46+
name: Build
47+
on: [push]
48+
49+
jobs:
50+
build-electron:
51+
name: Build Electron app - ${{ matrix.config.name }}
52+
runs-on: ${{ matrix.config.os }}
53+
strategy:
54+
fail-fast: false
55+
matrix:
56+
config:
57+
- name: "Windows"
58+
os: windows-2022
59+
- name: "Ubuntu"
60+
os: ubuntu-22.04
61+
- name: "macOS"
62+
os: macos-13
63+
64+
steps:
65+
- uses: actions/checkout@v4
66+
- uses: actions/setup-node@v4
67+
with:
68+
node-version: "20"
69+
70+
- name: Install dependencies on Ubuntu
71+
if: matrix.config.name == 'Ubuntu'
72+
run: |
73+
sudo apt-get update
74+
sudo apt-get install libarchive-tools rpm
75+
sudo snap install snapcraft --classic
76+
77+
- name: Install modules
78+
run: npm ci
79+
80+
- name: Build electron app
81+
id: build
82+
shell: bash
83+
timeout-minutes: 480
84+
run: npm run build
85+
86+
- name: Upload artifacts
87+
uses: actions/upload-artifact@v4
88+
with:
89+
include-hidden-files: true
90+
name: "electron-app-${{ matrix.config.name }}"
91+
path: "./release"
92+
```
93+
94+
</span>
95+
96+
:::
97+
4198
## Bundling
4299
When bundling your code for Electron using [Electron Vite](https://electron-vite.org) or Webpack,
43100
ensure that `node-llama-cpp` is not bundled, and is instead treated as an external module.

‎docs/guide/troubleshooting.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,3 +156,10 @@ please [open a new issue on GitHub](https://github.com/withcatai/node-llama-cpp/
156156
The common cause for this issue is when using the `Administrator` to run `npm install` and then trying to run the code with a different user.
157157

158158
Ensure you're not using the `Administrator` user for `npm install` nor to run the code.
159+
160+
## Getting an `EPERM: operation not permitted` Error on a Windows Machine When Building an Electron App
161+
`electron-builder` needs to create symlinks to perform the build process, which requires enabling Developer Mode on Windows.
162+
163+
To do that, go to `Settings > Update & Security > For developers` and enable `Developer mode`.
164+
165+
After that, delete the `.cache` folder under your user directory and try building the app again.

‎docs/public/robots.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
User-agent: *
2+
23
Sitemap: https://node-llama-cpp.withcat.ai/sitemap.xml

‎llama/CMakeLists.txt‎

Lines changed: 10 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,16 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Ap
2828
add_compile_options(-Wno-c++17-extensions)
2929
endif()
3030

31+
if(APPLE)
32+
set(CMAKE_SKIP_BUILD_RPATH FALSE)
33+
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
34+
set(CMAKE_BUILD_RPATH "@loader_path")
35+
set(CMAKE_INSTALL_RPATH "@loader_path")
36+
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
37+
else()
38+
set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
39+
endif()
40+
3141
include_directories(${NODE_ADDON_API_DIR} ${CMAKE_JS_INC})
3242

3343
add_subdirectory("llama.cpp")
@@ -39,41 +49,6 @@ unset(GPU_INFO_HEADERS)
3949
unset(GPU_INFO_SOURCES)
4050
unset(GPU_INFO_EXTRA_LIBS)
4151

42-
if (GGML_CUDA)
43-
cmake_minimum_required(VERSION 3.17)
44-
45-
find_package(CUDAToolkit)
46-
if (CUDAToolkit_FOUND)
47-
message(STATUS "Using CUDA for GPU info")
48-
49-
enable_language(CUDA)
50-
51-
list(APPEND GPU_INFO_HEADERS gpuInfo/cuda-gpu-info.h)
52-
list(APPEND GPU_INFO_SOURCES gpuInfo/cuda-gpu-info.cu)
53-
54-
add_compile_definitions(GPU_INFO_USE_CUDA)
55-
56-
if (GGML_STATIC)
57-
list(APPEND GPU_INFO_EXTRA_LIBS CUDA::cudart_static)
58-
else()
59-
list(APPEND GPU_INFO_EXTRA_LIBS CUDA::cudart)
60-
endif()
61-
62-
list(APPEND GPU_INFO_EXTRA_LIBS CUDA::cuda_driver)
63-
64-
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
65-
# copied from llama.cpp/CMakLists.txt under "if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)"
66-
if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
67-
set(CMAKE_CUDA_ARCHITECTURES "60;61;70;75")
68-
else()
69-
set(CMAKE_CUDA_ARCHITECTURES "52;61;70;75")
70-
endif()
71-
endif()
72-
else()
73-
message(FATAL_ERROR "CUDA was not found")
74-
endif()
75-
endif()
76-
7752
if (GGML_VULKAN OR GGML_KOMPUTE)
7853
find_package(Vulkan)
7954
if (Vulkan_FOUND)
@@ -94,67 +69,12 @@ if (GGML_VULKAN OR GGML_KOMPUTE)
9469
endif()
9570
endif()
9671

97-
if (GGML_HIPBLAS)
98-
list(APPEND CMAKE_PREFIX_PATH /opt/rocm)
99-
100-
if (NOT ${CMAKE_C_COMPILER_ID} MATCHES "Clang")
101-
message(WARNING "Only LLVM is supported for HIP, hint: CC=/opt/rocm/llvm/bin/clang")
102-
endif()
103-
if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
104-
message(WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++")
105-
endif()
106-
107-
find_package(hip)
108-
find_package(hipblas)
109-
find_package(rocblas)
110-
111-
if (${hipblas_FOUND} AND ${hip_FOUND})
112-
message(STATUS "Using HIP and hipBLAS for GPU info")
113-
add_compile_definitions(GPU_INFO_USE_HIPBLAS GPU_INFO_USE_CUDA)
114-
add_library(gpu-info-rocm OBJECT gpuInfo/cuda-gpu-info.cu gpuInfo/cuda-gpu-info.h)
115-
set_source_files_properties(gpuInfo/cuda-gpu-info.cu PROPERTIES LANGUAGE CXX)
116-
target_link_libraries(gpu-info-rocm PRIVATE hip::device PUBLIC hip::host roc::rocblas roc::hipblas)
117-
118-
list(APPEND GPU_INFO_EXTRA_LIBS gpu-info-rocm)
119-
else()
120-
message(FATAL_ERROR "hipBLAS or HIP was not found. Try setting CMAKE_PREFIX_PATH=/opt/rocm")
121-
endif()
122-
endif()
123-
124-
if (GGML_METAL)
125-
find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
126-
find_library(METAL_FRAMEWORK Metal REQUIRED)
127-
find_library(METALKIT_FRAMEWORK MetalKit REQUIRED)
128-
129-
message(STATUS "Using Metal for GPU info")
130-
list(APPEND GPU_INFO_HEADERS gpuInfo/metal-gpu-info.h)
131-
list(APPEND GPU_INFO_SOURCES gpuInfo/metal-gpu-info.mm)
132-
133-
add_compile_definitions(GPU_INFO_USE_METAL)
134-
135-
list(APPEND GPU_INFO_EXTRA_LIBS
136-
${FOUNDATION_LIBRARY}
137-
${METAL_FRAMEWORK}
138-
${METALKIT_FRAMEWORK}
139-
)
140-
endif()
141-
14272
list(REMOVE_DUPLICATES GPU_INFO_HEADERS)
14373
list(REMOVE_DUPLICATES GPU_INFO_SOURCES)
14474
list(REMOVE_DUPLICATES GPU_INFO_EXTRA_LIBS)
14575

14676
file(GLOB SOURCE_FILES "addon/*.cpp" "addon/**/*.cpp" ${GPU_INFO_SOURCES})
14777

148-
if(APPLE)
149-
set(CMAKE_SKIP_BUILD_RPATH FALSE)
150-
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
151-
set(CMAKE_BUILD_RPATH "@loader_path")
152-
set(CMAKE_INSTALL_RPATH "@loader_path")
153-
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
154-
else()
155-
set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
156-
endif()
157-
15878
add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC} ${GPU_INFO_HEADERS})
15979
set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
16080
target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB})

0 commit comments

Comments
 (0)