feat: move prebuilt fallback CUDA backend to an additional npm package

Author: giladgd

llama/CMakeLists.txt

@@ -86,11 +86,8 @@ endif()
 
 if (GGML_CUDA AND NOT DEFINED CMAKE_CUDA_ARCHITECTURES AND NOT NLC_GGML_NATIVE)
     find_package(CUDAToolkit)
-
     if (CUDAToolkit_FOUND AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL "13.0")
         set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real;90-real")
-    elseif (CUDAToolkit_FOUND AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.2")
-        set(CMAKE_CUDA_ARCHITECTURES "50-virtual;61-virtual;70-virtual;75-virtual;80-virtual;86-virtual;89-virtual")
     endif()
 endif()
 

package.json

@@ -228,12 +228,14 @@
     "@node-llama-cpp/linux-armv7l": "0.1.0",
     "@node-llama-cpp/linux-x64": "0.1.0",
     "@node-llama-cpp/linux-x64-cuda": "0.1.0",
+    "@node-llama-cpp/linux-x64-cuda-ext": "0.1.0",
     "@node-llama-cpp/linux-x64-vulkan": "0.1.0",
     "@node-llama-cpp/mac-arm64-metal": "0.1.0",
     "@node-llama-cpp/mac-x64": "0.1.0",
     "@node-llama-cpp/win-arm64": "0.1.0",
     "@node-llama-cpp/win-x64": "0.1.0",
     "@node-llama-cpp/win-x64-cuda": "0.1.0",
+    "@node-llama-cpp/win-x64-cuda-ext": "0.1.0",
     "@node-llama-cpp/win-x64-vulkan": "0.1.0"
   }
 }

packages/@node-llama-cpp/linux-x64-cuda-ext/.gitignore

@@ -0,0 +1 @@
+/dist

packages/@node-llama-cpp/linux-x64-cuda-ext/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Gilad S.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/@node-llama-cpp/linux-x64-cuda-ext/README.md

@@ -0,0 +1,5 @@
+# [`node-llama-cpp`](https://github.com/withcatai/node-llama-cpp)
+This is an extension of the [`@node-llama-cpp/linux-x64-cuda`](https://www.npmjs.com/package/@node-llama-cpp/linux-x64-cuda) package
+that provides a prebuilt binary for [`node-llama-cpp`](https://github.com/withcatai/node-llama-cpp) for Linux x64 with CUDA support.
+
+Do not install this package directly.

packages/@node-llama-cpp/linux-x64-cuda-ext/package-lock.json

@@ -0,0 +1,48 @@
+{
+  "name": "@node-llama-cpp/linux-x64-cuda-ext",
+  "version": "0.1.0",
+  "description": "Extension of @node-llama-cpp/linux-x64-cuda - prebuilt binary for node-llama-cpp for Linux x64 with CUDA support",
+  "main": "dist/index.js",
+  "type": "module",
+  "files": [
+    "dist/",
+    "bins/",
+    "package.json",
+    "README.md",
+    "LICENSE"
+  ],
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "node": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "os": ["linux"],
+  "cpu": ["x64"],
+  "libc": ["glibc"],
+  "scripts": {
+    "prebuild": "rimraf ./dist ./tsconfig.tsbuildinfo",
+    "build": "tsc --build tsconfig.json --force",
+    "prewatch": "rimraf ./dist ./tsconfig.tsbuildinfo",
+    "watch": "tsc --build tsconfig.json --watch --force",
+    "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/withcatai/node-llama-cpp.git"
+  },
+  "author": "Gilad S.",
+  "license": "MIT",
+  "preferUnplugged": true,
+  "bugs": {
+    "url": "https://github.com/withcatai/node-llama-cpp/issues"
+  },
+  "homepage": "https://node-llama-cpp.withcat.ai",
+  "devDependencies": {
+    "typescript": "^5.2.2"
+  }
+}

packages/@node-llama-cpp/linux-x64-cuda-ext/package.json

@@ -0,0 +1,14 @@
+import path from "path";
+import {fileURLToPath} from "url";
+import fs from "node:fs/promises";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const binsDir = path.join(__dirname, "..", "bins");
+const packageVersion: string = (JSON.parse(await fs.readFile(path.join(__dirname, "..", "package.json"), "utf8"))).version;
+
+export function getBinsDir() {
+    return {
+        binsDir,
+        packageVersion
+    };
+}

packages/@node-llama-cpp/linux-x64-cuda-ext/src/index.ts

@@ -0,0 +1,34 @@
+{
+    "compilerOptions": {
+        "lib": ["es2022"],
+        "module": "es2022",
+        "target": "es2022",
+        "esModuleInterop": true,
+        "noImplicitAny": true,
+        "noImplicitReturns": true,
+        "noImplicitThis": true,
+        "noImplicitOverride": true,
+        "removeComments": false,
+        "allowSyntheticDefaultImports": true,
+        "forceConsistentCasingInFileNames": true,
+        "noFallthroughCasesInSwitch": true,
+        "skipLibCheck": true,
+        "moduleResolution": "node",
+        "resolveJsonModule": false,
+        "strictNullChecks": true,
+        "isolatedModules": true,
+        "noEmit": false,
+        "outDir": "./dist",
+        "strict": true,
+        "sourceMap": false,
+        "composite": false,
+        "declaration": false,
+        "stripInternal": true
+    },
+    "files": [
+        "./src/index.ts"
+    ],
+    "include": [
+        "./src"
+    ]
+}

packages/@node-llama-cpp/linux-x64-cuda-ext/tsconfig.json

@@ -0,0 +1 @@
+/dist