Skip to content

Commit 2189055

Browse files
martindevansmartindevans
committed
Updated binaries, removed old CUDA11 and replaced 12.2 with 12.4
1 parent 50f2425 commit 2189055

13 files changed

+72
-256
lines changed

‎LLama.Examples/Examples/BatchedExecutorSimple.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,8 @@ await AnsiConsole.Live(table).StartAsync(async ctx =>
9797

9898
// A generic error, this is fatal and the batch can no longer be used. This should never occur and generally indicates
9999
// a bug in LLamaSharp, llama.cpp or a hardware error.
100-
if (decodeResult == DecodeResult.Error)
101-
throw new Exception("Unknown error occurred while inferring.");
100+
if (decodeResult != DecodeResult.Ok)
101+
throw new Exception($"Error occurred while inferring: {decodeResult}");
102102

103103
// After inference all of the conversations must be sampled before running inference again.
104104
foreach (var conversationData in conversations)

‎LLama.Unittest/LLavaWeightsTests.cs

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎LLama.Web/Common/ModelOptions.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,12 @@ public class ModelOptions
110110
/// <inheritdoc />
111111
public bool VocabOnly { get; set; }
112112

113+
/// <inheritdoc />
114+
public bool? OpOffload { get; set; }
115+
116+
/// <inheritdoc />
117+
public bool? SwaFull { get; set; }
118+
113119
/// <inheritdoc />
114120
public float? DefragThreshold { get; set; }
115121

‎LLama/Common/ModelParams.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,12 @@ public record ModelParams
112112
/// <inheritdoc />
113113
public bool VocabOnly { get; set; }
114114

115+
/// <inheritdoc />
116+
public bool? OpOffload { get; set; }
117+
118+
/// <inheritdoc />
119+
public bool? SwaFull { get; set; }
120+
115121
/// <summary>
116122
/// `Encoding` cannot be directly JSON serialized, instead store the name as a string which can
117123
/// </summary>

‎LLama/LLamaSharp.Runtime.targets

Lines changed: 46 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -76,37 +76,19 @@
7676
</None>
7777

7878

79-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/llama.dll">
80-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
81-
<Link>runtimes/win-x64/native/cuda11/llama.dll</Link>
82-
</None>
83-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/ggml-base.dll">
84-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
85-
<Link>runtimes/win-x64/native/cuda11/ggml-base.dll</Link>
86-
</None>
87-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/ggml.dll">
88-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
89-
<Link>runtimes/win-x64/native/cuda11/ggml.dll</Link>
90-
</None>
91-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/ggml-cuda.dll">
92-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
93-
<Link>runtimes/win-x64/native/cuda11/ggml-cuda.dll</Link>
94-
</None>
95-
96-
97-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/llama.dll">
79+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/llama.dll">
9880
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
9981
<Link>runtimes/win-x64/native/cuda12/llama.dll</Link>
10082
</None>
101-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/ggml-base.dll">
83+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/ggml-base.dll">
10284
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
10385
<Link>runtimes/win-x64/native/cuda12/ggml-base.dll</Link>
10486
</None>
105-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/ggml.dll">
87+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/ggml.dll">
10688
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
10789
<Link>runtimes/win-x64/native/cuda12/ggml.dll</Link>
10890
</None>
109-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/ggml-cuda.dll">
91+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/ggml-cuda.dll">
11092
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
11193
<Link>runtimes/win-x64/native/cuda12/ggml-cuda.dll</Link>
11294
</None>
@@ -218,43 +200,25 @@
218200
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
219201
<Link>runtimes/linux-arm64/native/libggml-cpu.so</Link>
220202
</None>
221-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libllava_shared.so">
222-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
223-
<Link>runtimes/linux-arm64/native/libllava_shared.so</Link>
224-
</None>
225-
226-
227-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libllama.so">
228-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
229-
<Link>runtimes/linux-x64/native/cuda11/libllama.so</Link>
230-
</None>
231-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libggml.so">
232-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
233-
<Link>runtimes/linux-x64/native/cuda11/libggml.so</Link>
234-
</None>
235-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libggml-base.so">
236-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
237-
<Link>runtimes/linux-x64/native/cuda11/libggml-base.so</Link>
238-
</None>
239-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libggml-cuda.so">
203+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libmtmd.so">
240204
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
241-
<Link>runtimes/linux-x64/native/cuda11/libggml-cuda.so</Link>
205+
<Link>runtimes/linux-arm64/native/libmtmd.so</Link>
242206
</None>
243207

244208

245-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/libllama.so">
209+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/libllama.so">
246210
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
247211
<Link>runtimes/linux-x64/native/cuda12/libllama.so</Link>
248212
</None>
249-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/libggml.so">
213+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/libggml.so">
250214
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
251215
<Link>runtimes/linux-x64/native/cuda12/libggml.so</Link>
252216
</None>
253-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/libggml-base.so">
217+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/libggml-base.so">
254218
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
255219
<Link>runtimes/linux-x64/native/cuda12/libggml-base.so</Link>
256220
</None>
257-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/libggml-cuda.so">
221+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/libggml-cuda.so">
258222
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
259223
<Link>runtimes/linux-x64/native/cuda12/libggml-cuda.so</Link>
260224
</None>
@@ -371,9 +335,9 @@
371335
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
372336
<Link>runtimes/osx-arm64/native/libllama.dylib</Link>
373337
</None>
374-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libllava_shared.dylib">
338+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libmtmd.dylib">
375339
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
376-
<Link>runtimes/osx-arm64/native/libllava_shared.dylib</Link>
340+
<Link>runtimes/osx-arm64/native/libmtmd.dylib</Link>
377341
</None>
378342
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/ggml-metal.metal">
379343
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -400,9 +364,9 @@
400364
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
401365
<Link>runtimes/osx-x64/native/libllama.dylib</Link>
402366
</None>
403-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64/libllava_shared.dylib">
367+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64/libmtmd.dylib">
404368
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
405-
<Link>runtimes/osx-x64/native/libllava_shared.dylib</Link>
369+
<Link>runtimes/osx-x64/native/libmtmd.dylib</Link>
406370
</None>
407371

408372
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64-rosetta2/libggml-base.dylib">
@@ -425,67 +389,59 @@
425389
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
426390
<Link>runtimes/osx-x64/native/rosetta2/libllama.dylib</Link>
427391
</None>
428-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64-rosetta2/libllava_shared.dylib">
392+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64-rosetta2/libmtmd.dylib">
429393
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
430-
<Link>runtimes/osx-x64/native/rosetta2/libllava_shared.dylib</Link>
394+
<Link>runtimes/osx-x64/native/rosetta2/libmtmd.dylib</Link>
431395
</None>
432396

433-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/noavx/llava_shared.dll">
434-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
435-
<Link>runtimes/win-x64/native/noavx/llava_shared.dll</Link>
436-
</None>
437-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/llava_shared.dll">
397+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/noavx/mtmd.dll">
438398
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
439-
<Link>runtimes/win-x64/native/avx/llava_shared.dll</Link>
399+
<Link>runtimes/win-x64/native/noavx/libmtmd.dll</Link>
440400
</None>
441-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/llava_shared.dll">
401+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/mtmd.dll">
442402
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
443-
<Link>runtimes/win-x64/native/avx2/llava_shared.dll</Link>
403+
<Link>runtimes/win-x64/native/avx/libmtmd.dll</Link>
444404
</None>
445-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/llava_shared.dll">
405+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/mtmd.dll">
446406
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
447-
<Link>runtimes/win-x64/native/avx512/llava_shared.dll</Link>
407+
<Link>runtimes/win-x64/native/avx2/libmtmd.dll</Link>
448408
</None>
449-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/llava_shared.dll">
409+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/mtmd.dll">
450410
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
451-
<Link>runtimes/win-x64/native/cuda11/llava_shared.dll</Link>
411+
<Link>runtimes/win-x64/native/avx512/libmtmd.dll</Link>
452412
</None>
453-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/llava_shared.dll">
413+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/mtmd.dll">
454414
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
455-
<Link>runtimes/win-x64/native/cuda12/llava_shared.dll</Link>
415+
<Link>runtimes/win-x64/native/cuda12/libmtmd.dll</Link>
456416
</None>
457-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/llava_shared.dll">
417+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/mtmd.dll">
458418
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
459-
<Link>runtimes/win-x64/native/vulkan/llava_shared.dll</Link>
419+
<Link>runtimes/win-x64/native/vulkan/libmtmd.dll</Link>
460420
</None>
461421

462-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/noavx/libllava_shared.so">
463-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
464-
<Link>runtimes/linux-x64/native/noavx/libllava_shared.so</Link>
465-
</None>
466-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/libllava_shared.so">
422+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/noavx/libmtmd.so">
467423
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
468-
<Link>runtimes/linux-x64/native/avx/libllava_shared.so</Link>
424+
<Link>runtimes/linux-x64/native/noavx/libmtmd.so</Link>
469425
</None>
470-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/libllava_shared.so">
426+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx/libmtmd.so">
471427
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
472-
<Link>runtimes/linux-x64/native/avx2/libllava_shared.so</Link>
428+
<Link>runtimes/linux-x64/native/avx/libmtmd.so</Link>
473429
</None>
474-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/libllava_shared.so">
430+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx2/libmtmd.so">
475431
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
476-
<Link>runtimes/linux-x64/native/avx512/libllava_shared.so</Link>
432+
<Link>runtimes/linux-x64/native/avx2/libmtmd.so</Link>
477433
</None>
478-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libllava_shared.so">
434+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/avx512/libmtmd.so">
479435
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
480-
<Link>runtimes/linux-x64/native/cuda11/libllava_shared.so</Link>
436+
<Link>runtimes/linux-x64/native/avx512/libmtmd.so</Link>
481437
</None>
482-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.2.0/libllava_shared.so">
438+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu12.4.0/libmtmd.so">
483439
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
484-
<Link>runtimes/linux-x64/native/cuda12/libllava_shared.so</Link>
440+
<Link>runtimes/linux-x64/native/cuda12/libmtmd.so</Link>
485441
</None>
486-
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libllava_shared.so">
442+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/vulkan/libmtmd.so">
487443
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
488-
<Link>runtimes/linux-x64/native/vulkan/libllava_shared.so</Link>
444+
<Link>runtimes/linux-x64/native/vulkan/libmtmd.so</Link>
489445
</None>
490446
</ItemGroup>
491447

@@ -513,8 +469,8 @@
513469
<Abi>x86</Abi>
514470
</AndroidNativeLibrary>
515471
<AndroidNativeLibrary Visible="false"
516-
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libllava_shared.so">
517-
<Link>runtimes/android-x86/native/libllava_shared.so</Link>
472+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libmtmd.so">
473+
<Link>runtimes/android-x86/native/libmtmd.so</Link>
518474
<Abi>x86</Abi>
519475
</AndroidNativeLibrary>
520476
</ItemGroup>
@@ -542,8 +498,8 @@
542498
<Abi>x86_64</Abi>
543499
</AndroidNativeLibrary>
544500
<AndroidNativeLibrary Visible="false"
545-
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libllava_shared.so">
546-
<Link>lib/x86_64/libllava_shared.so</Link>
501+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libmtmd.so">
502+
<Link>lib/x86_64/libmtmd.so</Link>
547503
<Abi>x86_64</Abi>
548504
</AndroidNativeLibrary>
549505
</ItemGroup>
@@ -571,8 +527,8 @@
571527
<Abi>arm64-v8a</Abi>
572528
</AndroidNativeLibrary>
573529
<AndroidNativeLibrary Visible="false"
574-
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libllava_shared.so">
575-
<Link>lib/arm64-v8a/libllava_shared.so</Link>
530+
Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libmtmd.so">
531+
<Link>lib/arm64-v8a/libmtmd.so</Link>
576532
<Abi>arm64-v8a</Abi>
577533
</AndroidNativeLibrary>
578534
</ItemGroup>

‎LLama/LLamaSharp.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
</ItemGroup>
5858

5959
<PropertyGroup>
60-
<BinaryReleaseId>ceda28ef8e310_v2</BinaryReleaseId>
60+
<BinaryReleaseId>a0374a67e2924f2e</BinaryReleaseId>
6161
</PropertyGroup>
6262

6363
<PropertyGroup>

‎LLama/Native/NativeApi.Memory.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ internal struct llama_memory_i;
1212
/// </summary>
1313
/// <param name="mem"></param>
1414
/// <param name="data"></param>
15-
public static extern void llama_memory_clear(IntPtr /* llama_memory_t */ mem, bool data);
15+
public static extern void llama_memory_clear(IntPtr /* llama_memory_t */ mem, [MarshalAs(UnmanagedType.U1)] bool data);
1616

1717
/// <summary>
1818
/// Removes all tokens that belong to the specified sequence and have positions in [p0, p1)

‎LLama/Native/SafeLLamaContextHandle.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -504,8 +504,6 @@ public Span<float> GetEmbeddingsSeq(LLamaSeqId seq)
504504

505505
unsafe
506506
{
507-
SafeLlamaModelHandle.
508-
509507
var embd = llama_get_embeddings_seq(this, seq);
510508
Debug.Assert(embd != null);
511509
return new Span<float>(embd, model.EmbeddingSize);

‎LLama/runtimes/build/LLamaSharp.Backend.Cuda11.Linux.nuspec

Lines changed: 0 additions & 34 deletions
This file was deleted.

0 commit comments

Comments
 (0)