feat: NUMA policy (#482)

* feat: NUMA policy
* feat(`inspect gpu` command): log prebuilt binaries and cloned source releases
* fix: level of some internal logs
* fix: upgrade to latest `lifecycle-utils`
* fix: various bugs

Author: giladgd

.github/workflows/build.yml

@@ -57,7 +57,7 @@ jobs:
       matrix:
         config:
           - name: "Windows for x64"
-            os: windows-2019
+            os: windows-2022
             artifact: "win-x64"
           - name: "Windows for Arm"
             os: windows-2022

.vitepress/utils/ensureLocalImage.ts

@@ -55,7 +55,7 @@ export async function ensureLocalImage(url: string, name: string, {
     if (resolvedImages.has(cacheKey))
         return resolvedImages.get(cacheKey)!;
 
-    return await withLock(cacheKey[0], cacheKey[1], async () => {
+    return await withLock([resolvedImages, ...cacheKey], async () => {
         if (resolvedImages.has(cacheKey))
             return resolvedImages.get(cacheKey)!;
 
@@ -185,7 +185,9 @@ function getFileExtension(format: keyof FormatEnum | undefined) {
 async function fetchWithRetry(url: string, retires: number = 5, waitTime: number = 1000 * 2) {
     for (let i = retires; i >= 0; i--) {
         try {
-            return await fetch(url);
+            return await fetch(url, {
+                redirect: "follow"
+            });
         } catch (err) {
             if (i === 0) {
                 console.error(`Failed to fetch image: ${url}`, err);

docs/guide/embedding.md

@@ -172,7 +172,7 @@ const documents = [
     "Cleaning the house is a good way to keep it tidy"
 ];
 
-const query = "Tell me a goegraphical fact";
+const query = "Tell me a nature geographical fact";
 const rankedDocuments = await context.rankAndSort(query, documents);
 
 const topDocument = rankedDocuments[0]!;
@@ -185,7 +185,7 @@ console.log("Ranked documents:", rankedDocuments);
 ```
 > This example will produce this output:
 > ```
-> query: Tell me a goegraphical fact
+> query: Tell me a nature geographical fact
 > Top document: Mount Everest is the tallest mountain in the world
 > Second document: The capital of France is Paris
 > ```

llama/addon/addon.cpp

@@ -196,6 +196,36 @@ Napi::Value addonLoadBackends(const Napi::CallbackInfo& info) {
     return info.Env().Undefined();
 }
 
+Napi::Value addonSetNuma(const Napi::CallbackInfo& info) {
+    const bool numaDisabled = info.Length() == 0
+        ? true
+        : info[0].IsBoolean()
+            ? !info[0].As<Napi::Boolean>().Value()
+            : false;
+
+    if (numaDisabled)
+        return info.Env().Undefined();
+
+    const auto numaType = info[0].IsString()
+        ? info[0].As<Napi::String>().Utf8Value()
+        : "";
+
+    if (numaType == "distribute") {
+        llama_numa_init(GGML_NUMA_STRATEGY_DISTRIBUTE);
+    } else if (numaType == "isolate") {
+        llama_numa_init(GGML_NUMA_STRATEGY_ISOLATE);
+    } else if (numaType == "numactl") {
+        llama_numa_init(GGML_NUMA_STRATEGY_NUMACTL);
+    } else if (numaType == "mirror") {
+        llama_numa_init(GGML_NUMA_STRATEGY_MIRROR);
+    } else {
+        Napi::Error::New(info.Env(), std::string("Invalid NUMA strategy \"") + numaType + "\"").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    return info.Env().Undefined();
+}
+
 Napi::Value addonInit(const Napi::CallbackInfo& info) {
     if (backendInitialized) {
         Napi::Promise::Deferred deferred = Napi::Promise::Deferred::New(info.Env());
@@ -255,6 +285,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
         Napi::PropertyDescriptor::Function("getSwapInfo", getSwapInfo),
         Napi::PropertyDescriptor::Function("getMemoryInfo", getMemoryInfo),
         Napi::PropertyDescriptor::Function("loadBackends", addonLoadBackends),
+        Napi::PropertyDescriptor::Function("setNuma", addonSetNuma),
         Napi::PropertyDescriptor::Function("init", addonInit),
         Napi::PropertyDescriptor::Function("dispose", addonDispose),
     });

package-lock.json

@@ -157,7 +157,7 @@
     "@types/yargs": "^17.0.33",
     "@vitest/coverage-v8": "^3.1.3",
     "@vitest/ui": "^3.1.3",
-    "electron": "^36.2.0",
+    "electron": "^37.2.4",
     "eslint": "^9.26.0",
     "eslint-import-resolver-typescript": "^4.3.4",
     "eslint-plugin-import": "^2.31.0",
@@ -178,12 +178,12 @@
     "typescript-eslint": "^8.32.0",
     "vite-node": "^3.1.3",
     "vitepress": "^1.6.3",
-    "vitepress-plugin-llms": "https://pkg.pr.new/vitepress-plugin-llms@51",
+    "vitepress-plugin-llms": "^1.7.2",
     "vitest": "^3.1.3",
     "zx": "^8.5.4"
   },
   "dependencies": {
-    "@huggingface/jinja": "^0.5.0",
+    "@huggingface/jinja": "^0.5.1",
     "async-retry": "^1.3.3",
     "bytes": "^3.1.2",
     "chalk": "^5.4.1",
@@ -197,7 +197,7 @@
     "ignore": "^7.0.4",
     "ipull": "^3.9.2",
     "is-unicode-supported": "^2.1.0",
-    "lifecycle-utils": "^2.0.1",
+    "lifecycle-utils": "^3.0.1",
     "log-symbols": "^7.0.0",
     "nanoid": "^5.1.5",
     "node-addon-api": "^8.3.1",

package.json

@@ -1,4 +1,5 @@
 import {Token} from "../types.js";
+import {LlamaNuma} from "./types.js";
 
 
 export type BindingModule = {
@@ -85,6 +86,7 @@ export type BindingModule = {
         total: number
     },
     init(): Promise<void>,
+    setNuma(numa?: LlamaNuma): void,
     loadBackends(forceLoadLibrariesSearchPath?: string): void,
     dispose(): Promise<void>
 };
@@ -159,15 +161,15 @@ export type AddonContext = {
 };
 
 export type BatchLogitIndex = number & {
-    __batchLogitIndex: never
+    readonly __batchLogitIndex: never
 };
 
 export type AddonGrammar = {
     isTextCompatible(testText: string): boolean
 };
 
 export type AddonGrammarEvaluationState = "AddonGrammarEvaluationState" & {
-    __brand: never
+    readonly __brand: never
 };
 
 export type AddonSampler = {