Skip to content

Commit 2105dc3

Browse files
nipeonenipeone
committed
add support for linux-arm64
1 parent d97fbb3 commit 2105dc3

File tree

5 files changed

+91
-26
lines changed

5 files changed

+91
-26
lines changed

‎.github/workflows/compile.yml

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,25 @@ jobs:
2828
include:
2929
- build: 'noavx'
3030
defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF'
31+
os: ubuntu-20.04
32+
arch: x64
3133
- build: 'avx2'
3234
defines: ''
35+
os: ubuntu-20.04
36+
arch: x64
3337
- build: 'avx'
3438
defines: '-DGGML_AVX2=OFF'
39+
os: ubuntu-20.04
40+
arch: x64
3541
- build: 'avx512'
3642
defines: '-DGGML_AVX512=ON'
37-
runs-on: ubuntu-20.04
43+
os: ubuntu-20.04
44+
arch: x64
45+
- build: 'aarch64'
46+
defines: '-DGGML_NATIVE=OFF -DGGML_CPU_AARCH64=ON -DGGML_CPU_ARM_ARCH=generic'
47+
os: ubuntu-20.04-arm
48+
arch: arm64
49+
runs-on: ${{ matrix.os }}
3850
steps:
3951
- uses: actions/checkout@v4
4052
with:
@@ -52,28 +64,28 @@ jobs:
5264
- uses: actions/upload-artifact@v4
5365
with:
5466
path: ./build/bin/libllama.so
55-
name: llama-bin-linux-${{ matrix.build }}-x64.so
67+
name: llama-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
5668
if-no-files-found: error
5769
- uses: actions/upload-artifact@v4
5870
with:
5971
path: ./build/bin/libggml.so
60-
name: ggml-bin-linux-${{ matrix.build }}-x64.so
72+
name: ggml-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
6173
if-no-files-found: error
6274
- uses: actions/upload-artifact@v4
6375
with:
6476
path: ./build/bin/libggml-base.so
65-
name: ggml-base-bin-linux-${{ matrix.build }}-x64.so
77+
name: ggml-base-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
6678
if-no-files-found: error
6779
- uses: actions/upload-artifact@v4
6880
with:
6981
path: ./build/bin/libggml-cpu.so
70-
name: ggml-cpu-bin-linux-${{ matrix.build }}-x64.so
82+
name: ggml-cpu-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
7183
if-no-files-found: error
7284
- name: Upload Llava
7385
uses: actions/upload-artifact@v4
7486
with:
7587
path: ./build/bin/libllava_shared.so
76-
name: llava-bin-linux-${{ matrix.build }}-x64.so
88+
name: llava-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
7789
if-no-files-found: error
7890

7991
compile-musl:
@@ -601,7 +613,7 @@ jobs:
601613
- name: Rearrange Files
602614
run: |
603615
# Make all directories at once
604-
mkdir --parents deps/{noavx,avx,avx2,avx512,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
616+
mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
605617
606618
# Linux
607619
cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so deps/noavx/libggml.so
@@ -628,6 +640,13 @@ jobs:
628640
cp artifacts/llama-bin-linux-avx512-x64.so/libllama.so deps/avx512/libllama.so
629641
cp artifacts/llava-bin-linux-avx512-x64.so/libllava_shared.so deps/avx512/libllava_shared.so
630642
643+
# Arm64
644+
cp artifacts/ggml-bin-linux-aarch64-arm64.so/libggml.so deps/linux-arm64/libggml.so
645+
cp artifacts/ggml-base-bin-linux-aarch64-arm64.so/libggml-base.so deps/linux-arm64/libggml-base.so
646+
cp artifacts/ggml-cpu-bin-linux-aarch64-arm64.so/libggml-cpu.so deps/linux-arm64/libggml-cpu.so
647+
cp artifacts/llama-bin-linux-aarch64-arm64.so/libllama.so deps/linux-arm64/libllama.so
648+
cp artifacts/llava-bin-linux-aarch64-arm64.so/libllava_shared.so deps/linux-arm64/libllava_shared.so
649+
631650
# Musl
632651
cp artifacts/ggml-bin-musl-noavx-x64.so/libggml.so deps/musl-noavx/libggml.so
633652
cp artifacts/ggml-base-bin-musl-noavx-x64.so/libggml-base.so deps/musl-noavx/libggml-base.so

‎LLama/LLamaSharp.Runtime.targets

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,24 @@
202202
</None>
203203

204204

205+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libllama.so">
206+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
207+
<Link>runtimes/linux-arm64/native/libllama.so</Link>
208+
</None>
209+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml.so">
210+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
211+
<Link>runtimes/linux-arm64/native/libggml.so</Link>
212+
</None>
213+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml-base.so">
214+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
215+
<Link>runtimes/linux-arm64/native/libggml-base.so</Link>
216+
</None>
217+
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml-cpu.so">
218+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
219+
<Link>runtimes/linux-arm64/native/libggml-cpu.so</Link>
220+
</None>
221+
222+
205223
<None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libllama.so">
206224
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
207225
<Link>runtimes/linux-x64/native/cuda11/libllama.so</Link>

‎LLama/Native/Load/NativeLibraryUtils.cs

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -88,19 +88,28 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib
8888
// On other platforms (Windows, Linux), we need to load the CPU backend from the specified AVX level directory
8989
// We are using the AVX level supplied by NativeLibraryConfig, which automatically detects the highest supported AVX level for us
9090

91-
// ggml-cpu
92-
dependencyPaths.Add(Path.Combine(
93-
$"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}",
94-
$"{libPrefix}ggml-cpu{ext}"
95-
));
96-
97-
// ggml-cuda
98-
if (library.Metadata.UseCuda)
99-
dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cuda{ext}"));
100-
101-
// ggml-vulkan
102-
if (library.Metadata.UseVulkan)
103-
dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-vulkan{ext}"));
91+
if (os == "linux-arm64"){
92+
dependencyPaths.Add(Path.Combine(
93+
$"runtimes/{os}/native",
94+
$"{libPrefix}ggml-cpu{ext}"
95+
));
96+
}
97+
else{
98+
// ggml-cpu
99+
dependencyPaths.Add(Path.Combine(
100+
$"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}",
101+
$"{libPrefix}ggml-cpu{ext}"
102+
));
103+
104+
// ggml-cuda
105+
if (library.Metadata.UseCuda)
106+
dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cuda{ext}"));
107+
108+
// ggml-vulkan
109+
if (library.Metadata.UseVulkan)
110+
dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-vulkan{ext}"));
111+
}
112+
104113
}
105114
}
106115

@@ -218,6 +227,13 @@ public static void GetPlatformPathParts(OSPlatform platform, out string os, out
218227

219228
if (platform == OSPlatform.Linux)
220229
{
230+
if(System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported){
231+
// linux arm64
232+
os = "linux-arm64";
233+
fileExtension = ".so";
234+
libPrefix = "lib";
235+
return;
236+
}
221237
if(RuntimeInformation.RuntimeIdentifier.ToLower().StartsWith("alpine"))
222238
{
223239
// alpine linux distro

‎LLama/Native/Load/NativeLibraryWithAvx.cs

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,17 @@ public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaL
5050
private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback)
5151
{
5252
NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
53-
var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel);
54-
if (!string.IsNullOrEmpty(avxStr))
55-
avxStr += "/";
56-
var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
57-
return relativePath;
53+
if (os != "linux-arm64"){
54+
var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel);
55+
if (!string.IsNullOrEmpty(avxStr))
56+
avxStr += "/";
57+
var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
58+
return relativePath;
59+
} else {
60+
var relativePath = $"runtimes/{os}/native/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
61+
return relativePath;
62+
}
63+
5864
}
5965
}
6066
#endif

‎LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,13 @@
6666
<file src="runtimes/deps/avx512/libllama.so" target="runtimes\linux-x64\native\avx512\libllama.so" />
6767
<file src="runtimes/deps/avx512/libllava_shared.so" target="runtimes\linux-x64\native\avx512\libllava_shared.so" />
6868

69-
<file src="runtimes/deps/musl-noavx/libggml.so" target="runtimes\linux-musl-x64\native\noavx\libggml.so" />
69+
<file src="runtimes/deps/linux-arm64/libggml.so" target="runtimes\linux-arm64\native\libggml.so" />
70+
<file src="runtimes/deps/linux-arm64/libggml-base.so" target="runtimes\linux-arm64\native\libggml-base.so" />
71+
<file src="runtimes/deps/linux-arm64/libggml-cpu.so" target="runtimes\linux-arm64\native\libggml-cpu.so" />
72+
<file src="runtimes/deps/linux-arm64/libllama.so" target="runtimes\linux-arm64\native\libllama.so" />
73+
<file src="runtimes/deps/linux-arm64/libllava_shared.so" target="runtimes\linux-arm64\native\libllava_shared.so" />
74+
75+
<file src="runtimes/deps/musl-noavx/libggml.so" target="runtimes\linux-musl-x64\native\noavx\libggml.so" />
7076
<file src="runtimes/deps/musl-noavx/libggml-base.so" target="runtimes\linux-musl-x64\native\noavx\libggml-base.so" />
7177
<file src="runtimes/deps/musl-noavx/libggml-cpu.so" target="runtimes\linux-musl-x64\native\noavx\libggml-cpu.so" />
7278
<file src="runtimes/deps/musl-noavx/libllama.so" target="runtimes\linux-musl-x64\native\noavx\libllama.so" />

0 commit comments

Comments
 (0)