Skip to content

Commit 9864bfc

Browse files
compiladecompilade
committed
Merge branch 'master' into compilade/mamba2
2 parents e94f393 + dad5c44 commit 9864bfc

File tree

581 files changed

+160637
-39072
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

581 files changed

+160637
-39072
lines changed

‎.devops/intel.Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
ARG ONEAPI_VERSION=2025.0.0-0-devel-ubuntu22.04
1+
ARG ONEAPI_VERSION=2025.1.1-0-devel-ubuntu24.04
22

33
## Build Image
44

‎.devops/musa.Dockerfile

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
ARG UBUNTU_VERSION=22.04
22
# This needs to generally match the container host's environment.
3-
ARG MUSA_VERSION=rc3.1.1
3+
ARG MUSA_VERSION=rc4.0.1
44
# Target the MUSA build image
5-
ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
5+
ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-devel-ubuntu${UBUNTU_VERSION}
66

7-
ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
7+
ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-runtime-ubuntu${UBUNTU_VERSION}
88

99
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
1010

@@ -21,21 +21,14 @@ RUN apt-get update && \
2121
libcurl4-openssl-dev \
2222
libgomp1
2323

24-
COPY requirements.txt requirements.txt
25-
COPY requirements requirements
26-
27-
RUN pip install --upgrade pip setuptools wheel \
28-
&& pip install -r requirements.txt
29-
3024
WORKDIR /app
3125

3226
COPY . .
3327

34-
# Use the default MUSA archs if not specified
3528
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \
3629
export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \
3730
fi && \
38-
cmake -B build -DGGML_NATIVE=OFF -DGGML_MUSA=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON ${CMAKE_ARGS} -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined . && \
31+
cmake -B build -DGGML_NATIVE=OFF -DGGML_MUSA=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DLLAMA_BUILD_TESTS=OFF ${CMAKE_ARGS} -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined . && \
3932
cmake --build build --config Release -j$(nproc)
4033

4134
RUN mkdir -p /app/lib && \

‎.editorconfig

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,23 +21,23 @@ indent_style = tab
2121
[prompts/*.txt]
2222
insert_final_newline = unset
2323

24-
[examples/server/public/*]
24+
[tools/server/public/*]
2525
indent_size = 2
2626

27-
[examples/server/public/deps_*]
27+
[tools/server/public/deps_*]
2828
trim_trailing_whitespace = unset
2929
indent_style = unset
3030
indent_size = unset
3131

32-
[examples/server/deps_*]
32+
[tools/server/deps_*]
3333
trim_trailing_whitespace = unset
3434
indent_style = unset
3535
indent_size = unset
3636

3737
[examples/llama.swiftui/llama.swiftui.xcodeproj/*]
3838
indent_style = tab
3939

40-
[examples/cvector-generator/*.txt]
40+
[tools/cvector-generator/*.txt]
4141
trim_trailing_whitespace = unset
4242
insert_final_newline = unset
4343

@@ -48,3 +48,7 @@ end_of_line = unset
4848
charset = unset
4949
trim_trailing_whitespace = unset
5050
insert_final_newline = unset
51+
52+
[vendor/miniaudio/miniaudio.h]
53+
trim_trailing_whitespace = unset
54+
insert_final_newline = unset

‎.flake8

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
max-line-length = 125
33
ignore = E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503
44
exclude =
5-
# Do not traverse examples
5+
# Do not traverse examples and tools
66
examples,
7+
tools,
78
# Do not include package initializers
89
__init__.py,
910
# No need to traverse our git directory

‎.github/actions/get-tag-name/action.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: "Determine tag name"
2+
description: "Determine the tag name to use for a release"
3+
outputs:
4+
name:
5+
description: "The name of the tag"
6+
value: ${{ steps.tag.outputs.name }}
7+
8+
runs:
9+
using: "composite"
10+
steps:
11+
- name: Determine tag name
12+
id: tag
13+
shell: bash
14+
run: |
15+
BUILD_NUMBER="$(git rev-list --count HEAD)"
16+
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
17+
if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
18+
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
19+
else
20+
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
21+
echo "name=${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}" >> $GITHUB_OUTPUT
22+
fi

‎.github/actions/windows-setup-cuda/action.yml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: "Windows - Setup CUDA Toolkit"
2+
description: "Setup CUDA Toolkit for Windows"
3+
inputs:
4+
cuda_version:
5+
description: "CUDA toolkit version"
6+
required: true
7+
8+
runs:
9+
using: "composite"
10+
steps:
11+
- name: Install Cuda Toolkit 11.7
12+
if: ${{ inputs.cuda_version == '11.7' }}
13+
shell: pwsh
14+
run: |
15+
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
16+
choco install unzip -y
17+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-11.7.99-archive.zip"
18+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-11.7.99-archive.zip"
19+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-11.7.99-archive.zip"
20+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-11.7.4.6-archive.zip"
21+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-11.7.91-archive.zip"
22+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
23+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
24+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
25+
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
26+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
27+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
28+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
29+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\libcublas-windows-x86_64-11.7.4.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
30+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvtx-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
31+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\visual_studio_integration-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
32+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvprof-windows-x86_64-11.7.101-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
33+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cccl-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
34+
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
35+
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
36+
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
37+
echo "CUDA_PATH_V11_7=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
38+
39+
- name: Install Cuda Toolkit 12.4
40+
if: ${{ inputs.cuda_version == '12.4' }}
41+
shell: pwsh
42+
run: |
43+
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
44+
choco install unzip -y
45+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.4.127-archive.zip"
46+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-12.4.131-archive.zip"
47+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-12.4.127-archive.zip"
48+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-12.4.5.8-archive.zip"
49+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-12.4.127-archive.zip"
50+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-12.4.127-archive.zip"
51+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
52+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
53+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
54+
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
55+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
56+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
57+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
58+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\libcublas-windows-x86_64-12.4.5.8-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
59+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvtx-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
60+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_profiler_api-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
61+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\visual_studio_integration-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
62+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvprof-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
63+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cccl-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
64+
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
65+
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
66+
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
67+
echo "CUDA_PATH_V12_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8

‎.github/actions/windows-setup-curl/action.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ inputs:
55
description: 'CURL version'
66
required: false
77
default: '8.6.0_6'
8+
architecture:
9+
description: 'Architecture of the libcurl to download'
10+
required: false
11+
default: 'win64'
812
outputs:
913
curl_path:
1014
description: "Path to the downloaded libcurl"
@@ -18,8 +22,9 @@ runs:
1822
shell: powershell
1923
env:
2024
CURL_VERSION: ${{ inputs.curl_version }}
25+
ARCHITECTURE: ${{ inputs.architecture }}
2126
run: |
22-
curl.exe -o $env:RUNNER_TEMP/curl.zip -L "https://curl.se/windows/dl-${env:CURL_VERSION}/curl-${env:CURL_VERSION}-win64-mingw.zip"
27+
curl.exe -o $env:RUNNER_TEMP/curl.zip -L "https://curl.se/windows/dl-${env:CURL_VERSION}/curl-${env:CURL_VERSION}-${env:ARCHITECTURE}-mingw.zip"
2328
mkdir $env:RUNNER_TEMP/libcurl
2429
tar.exe -xvf $env:RUNNER_TEMP/curl.zip --strip-components=1 -C $env:RUNNER_TEMP/libcurl
2530
echo "curl_path=$env:RUNNER_TEMP/libcurl" >> $env:GITHUB_OUTPUT

‎.github/labeler.yml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,9 @@ build:
4545
- CMakePresets.json
4646
examples:
4747
- changed-files:
48-
- any-glob-to-any-file: examples/**
48+
- any-glob-to-any-file:
49+
- examples/**
50+
- tools/**
4951
devops:
5052
- changed-files:
5153
- any-glob-to-any-file:
@@ -70,7 +72,7 @@ android:
7072
server:
7173
- changed-files:
7274
- any-glob-to-any-file:
73-
- examples/server/**
75+
- tools/server/**
7476
ggml:
7577
- changed-files:
7678
- any-glob-to-any-file:
@@ -84,3 +86,10 @@ nix:
8486
embedding:
8587
- changed-files:
8688
- any-glob-to-any-file: examples/embedding/
89+
90+
Ascend NPU:
91+
- changed-files:
92+
- any-glob-to-any-file:
93+
- ggml/include/ggml-cann.h
94+
- ggml/src/ggml-cann/**
95+
- docs/backend/CANN.md

‎.github/workflows/bench.yml.disabled

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ on:
2727
push:
2828
branches:
2929
- master
30-
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'examples/server/*.h*', 'examples/server/*.cpp']
30+
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'tools/server/*.h*', 'tools/server/*.cpp']
3131
pull_request_target:
3232
types: [opened, synchronize, reopened]
33-
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'examples/server/*.h*', 'examples/server/*.cpp']
33+
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.cpp', 'ggml-quants.c', '**/*.cu', 'tools/server/*.h*', 'tools/server/*.cpp']
3434
schedule:
3535
- cron: '04 2 * * *'
3636

@@ -69,7 +69,7 @@ jobs:
6969
- name: Install python env
7070
id: pipenv
7171
run: |
72-
cd examples/server/bench
72+
cd tools/server/bench
7373
python3 -m venv venv
7474
source venv/bin/activate
7575
pip install -r requirements.txt
@@ -79,7 +79,7 @@ jobs:
7979
run: |
8080
wget --quiet https://github.com/prometheus/prometheus/releases/download/v2.51.0/prometheus-2.51.0.linux-amd64.tar.gz
8181
tar xzf prometheus*.tar.gz --strip-components=1
82-
./prometheus --config.file=examples/server/bench/prometheus.yml &
82+
./prometheus --config.file=tools/server/bench/prometheus.yml &
8383
while ! nc -z localhost 9090; do
8484
sleep 0.1
8585
done
@@ -92,7 +92,7 @@ jobs:
9292
- name: Install k6 and xk6-sse
9393
id: k6_installation
9494
run: |
95-
cd examples/server/bench
95+
cd tools/server/bench
9696
go install go.k6.io/xk6/cmd/xk6@latest
9797
xk6 build master \
9898
--with github.com/phymbert/xk6-sse
@@ -116,7 +116,7 @@ jobs:
116116
- name: Download the dataset
117117
id: download_dataset
118118
run: |
119-
cd examples/server/bench
119+
cd tools/server/bench
120120
wget --quiet https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
121121

122122
- name: Server bench
@@ -126,7 +126,7 @@ jobs:
126126
run: |
127127
set -eux
128128

129-
cd examples/server/bench
129+
cd tools/server/bench
130130
source venv/bin/activate
131131
python bench.py \
132132
--runner-label ${{ env.RUNNER_LABEL }} \
@@ -157,9 +157,9 @@ jobs:
157157
name: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }}
158158
compression-level: 9
159159
path: |
160-
examples/server/bench/*.jpg
161-
examples/server/bench/*.json
162-
examples/server/bench/*.log
160+
tools/server/bench/*.jpg
161+
tools/server/bench/*.json
162+
tools/server/bench/*.log
163163

164164
- name: Commit status
165165
uses: Sibz/github-status-action@v1
@@ -178,17 +178,17 @@ jobs:
178178
with:
179179
client_id: ${{secrets.IMGUR_CLIENT_ID}}
180180
path: |
181-
examples/server/bench/prompt_tokens_seconds.jpg
182-
examples/server/bench/predicted_tokens_seconds.jpg
183-
examples/server/bench/kv_cache_usage_ratio.jpg
184-
examples/server/bench/requests_processing.jpg
181+
tools/server/bench/prompt_tokens_seconds.jpg
182+
tools/server/bench/predicted_tokens_seconds.jpg
183+
tools/server/bench/kv_cache_usage_ratio.jpg
184+
tools/server/bench/requests_processing.jpg
185185

186186
- name: Extract mermaid
187187
id: set_mermaid
188188
run: |
189189
set -eux
190190

191-
cd examples/server/bench
191+
cd tools/server/bench
192192
PROMPT_TOKENS_SECONDS=$(cat prompt_tokens_seconds.mermaid)
193193
echo "PROMPT_TOKENS_SECONDS<<EOF" >> $GITHUB_ENV
194194
echo "$PROMPT_TOKENS_SECONDS" >> $GITHUB_ENV

0 commit comments

Comments
 (0)