fix: prebuilt binaries CUDA 13 support

Author: giladgd

.github/workflows/build.yml

@@ -56,15 +56,18 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - name: "Windows for x64"
+          - name: "Windows (1)"
             os: windows-2022
-            artifact: "win-x64"
-          - name: "Windows for Arm"
+            artifact: "win-1"
+          - name: "Windows (2)"
             os: windows-2022
-            artifact: "win-arm"
-          - name: "Ubuntu"
+            artifact: "win-2"
+          - name: "Ubuntu (1)"
+            os: ubuntu-22.04
+            artifact: "linux-1"
+          - name: "Ubuntu (2)"
             os: ubuntu-22.04
-            artifact: "linux"
+            artifact: "linux-2"
           - name: "macOS x64"
             os: macos-13
             artifact: "mac-x64"
@@ -98,7 +101,7 @@ jobs:
           choco install ninja
 
       - name: Install dependencies on Ubuntu
-        if: matrix.config.name == 'Ubuntu'
+        if: matrix.config.name == 'Ubuntu (1)' || matrix.config.name == 'Ubuntu (2)'
         run: |
           sudo apt-get update
           sudo apt-get install ninja-build cmake libtbb-dev g++-aarch64-linux-gnu gcc-aarch64-linux-gnu g++-arm-linux-gnueabihf gcc-arm-linux-gnueabihf
@@ -109,24 +112,40 @@ jobs:
           which arm-linux-gnueabihf-gcc
           which arm-linux-gnueabihf-g++
 
-      - name: Install Cuda on Windows for x64
-        if: matrix.config.name == 'Windows for x64'
+      - name: Install Cuda 12.4 on Windows (1)
+        if: matrix.config.name == 'Windows (1)'
         uses: Jimver/[email protected]
         with:
           cuda: '12.4.0'
           method: 'network'
           sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
           use-local-cache: false
 
-      - name: Install Cuda on Ubuntu
-        if: matrix.config.name == 'Ubuntu'
+      - name: Install Cuda 13.0 on Windows (2)
+        if: matrix.config.name == 'Windows (2)'
+        uses: Jimver/[email protected]
+        with:
+          cuda: '13.0.0'
+          method: 'network'
+          sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
+          use-local-cache: false
+
+      - name: Install Cuda 12.4 on Ubuntu
+        if: matrix.config.name == 'Ubuntu (1)'
         uses: Jimver/[email protected]
         with:
           cuda: '12.4.0'
           method: 'network'
 
-      - name: Install Vulkan SDK on Windows for x64
-        if: matrix.config.name == 'Windows for x64'
+      - name: Install Cuda 13.0 on Ubuntu
+        if: matrix.config.name == 'Ubuntu (2)'
+        uses: Jimver/[email protected]
+        with:
+          cuda: '13.0.0'
+          method: 'network'
+
+      - name: Install Vulkan SDK on Windows (1)
+        if: matrix.config.name == 'Windows (1)'
         shell: powershell
         env:
           VULKAN_VERSION: 1.3.261.1
@@ -137,7 +156,7 @@ jobs:
           Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
 
       - name: Install Vulkan SDK on Ubuntu
-        if: matrix.config.name == 'Ubuntu'
+        if: matrix.config.name == 'Ubuntu (1)'
         run: |
           wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
           sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
@@ -205,18 +224,21 @@ jobs:
           }
           
           // build binaries
-          if (process.env.ARTIFACT_NAME === "win-x64") {
+          if (process.env.ARTIFACT_NAME === "win-1") {
             await buildBinary("x64", ["--gpu", "false"]);
             await buildBinary("x64", ["--gpu", "cuda"]);
             await buildBinary("x64", ["--gpu", "vulkan"]);
-          } else if (process.env.ARTIFACT_NAME === "win-arm") {
+          } else if (process.env.ARTIFACT_NAME === "win-2") {
             await buildBinary("arm64", ["--gpu", "false"], windowsOnArmNodeVersion);
-          } else if (process.env.ARTIFACT_NAME === "linux") {
+            await buildBinary("x64", ["--gpu", "cuda"]);
+          } else if (process.env.ARTIFACT_NAME === "linux-1") {
             await buildBinary("x64", ["--gpu", "false"]);
             await buildBinary("x64", ["--gpu", "cuda"]);
             await buildBinary("x64", ["--gpu", "vulkan"]);
             await buildBinary("arm64", ["--gpu", "false"]);
             await buildBinary("armv7l", ["--gpu", "false"]);
+          } else if (process.env.ARTIFACT_NAME === "linux-2") {
+            await buildBinary("x64", ["--gpu", "cuda"]);
           } else if (process.env.ARTIFACT_NAME === "mac-x64") {
             await buildBinary("x64", ["--gpu", "false"]);
           } else if (process.env.ARTIFACT_NAME === "mac-arm64") {
@@ -233,6 +255,18 @@ jobs:
             );
           }
           
+          if (process.env.ARTIFACT_NAME === "win-2") {
+            await fs.move(
+              path.join(llamaBinsDirectoryPath, "win-x64-cuda"),
+              path.join(llamaBinsDirectoryPath, "win-x64-cuda-2")
+            );
+          } else if (process.env.ARTIFACT_NAME === "linux-2") {
+            await fs.move(
+              path.join(llamaBinsDirectoryPath, "linux-x64-cuda"),
+              path.join(llamaBinsDirectoryPath, "linux-x64-cuda-2")
+            );
+          }
+          
           await $`echo "Built binaries:"`;
           await $`ls bins`;
           
@@ -494,6 +528,14 @@ jobs:
           mv artifacts/bins-*/* bins/
           mv artifacts/build dist/
           
+          mkdir -p ./bins/win-x64-cuda/bins/win-x64-cuda/fallback
+          mv ./bins/win-x64-cuda-2/bins/win-x64-cuda/ggml-cuda.dll bins/win-x64-cuda/bins/win-x64-cuda/fallback/ggml-cuda.dll
+          rm -rf ./bins/win-x64-cuda-2
+          
+          mkdir -p ./bins/linux-x64-cuda/bins/linux-x64-cuda/fallback
+          mv ./bins/linux-x64-cuda-2/bins/linux-x64-cuda/libggml-cuda.so bins/linux-x64-cuda/bins/linux-x64-cuda/fallback/libggml-cuda.so
+          rm -rf ./bins/linux-x64-cuda-2
+          
           cp -r artifacts/llama.cpp/llama.cpp/grammars llama/grammars
           
           rm -f ./llama/binariesGithubRelease.json

src/bindings/Llama.ts

@@ -107,9 +107,17 @@ export class Llama {
         }
 
         bindings.loadBackends();
-        const loadedGpu = bindings.getGpuType();
-        if (loadedGpu == null || (loadedGpu === false && buildGpu !== false))
-            bindings.loadBackends(path.dirname(bindingPath));
+        let loadedGpu = bindings.getGpuType();
+        if (loadedGpu == null || (loadedGpu === false && buildGpu !== false)) {
+            const backendsPath = path.dirname(bindingPath);
+            const fallbackBackendsDir = path.join(backendsPath, "fallback");
+
+            bindings.loadBackends(backendsPath);
+
+            loadedGpu = bindings.getGpuType();
+            if (loadedGpu == null || (loadedGpu === false && buildGpu !== false))
+                bindings.loadBackends(fallbackBackendsDir);
+        }
 
         bindings.ensureGpuDeviceIsSupported();
 

src/bindings/utils/testBindingBinary.ts

@@ -286,9 +286,17 @@ if (process.env.TEST_BINDING_CP === "true" && (process.parentPort != null || pro
                     throw new Error("Binding binary is not loaded");
 
                 binding.loadBackends();
-                const loadedGpu = binding.getGpuType();
-                if (loadedGpu == null || (loadedGpu === false && message.gpu !== false))
-                    binding.loadBackends(path.dirname(path.resolve(message.bindingBinaryPath)));
+                let loadedGpu = binding.getGpuType();
+                if (loadedGpu == null || (loadedGpu === false && message.gpu !== false)) {
+                    const backendsPath = path.dirname(path.resolve(message.bindingBinaryPath));
+                    const fallbackBackendsDir = path.join(backendsPath, "fallback");
+
+                    binding.loadBackends(backendsPath);
+
+                    loadedGpu = binding.getGpuType();
+                    if (loadedGpu == null || (loadedGpu === false && message.gpu !== false))
+                        binding.loadBackends(fallbackBackendsDir);
+                }
 
                 await binding.init();
                 binding.getGpuVramInfo();