Skip to content

Commit 8408090

Browse files
NexesenexNexesenex
authored
Merge b3579
b3579
2 parents 6131e6a + fc4ca27 commit 8408090

File tree

12 files changed

+1244
-33
lines changed

12 files changed

+1244
-33
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ jobs:
129129
130130
- name: Server bench
131131
id: server_bench
132+
env:
133+
HEAD_REF: ${{ github.head_ref || github.ref_name }}
132134
run: |
133135
set -eux
134136
@@ -137,7 +139,7 @@ jobs:
137139
python bench.py \
138140
--runner-label ${{ env.RUNNER_LABEL }} \
139141
--name ${{ github.job }} \
140-
--branch ${{ github.head_ref || github.ref_name }} \
142+
--branch $HEAD_REF \
141143
--commit ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha }} \
142144
--scenario script.js \
143145
--duration ${{ github.event.inputs.duration || env.DURATION }} \

‎.github/workflows/build.yml‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
sysctl -a
4848
mkdir build
4949
cd build
50-
cmake -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF ..
50+
cmake -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF ..
5151
cmake --build . --config Release -j $(sysctl -n hw.logicalcpu)
5252
5353
- name: Test
@@ -105,7 +105,7 @@ jobs:
105105
sysctl -a
106106
# Metal is disabled due to intermittent failures with Github runners not having a GPU:
107107
# https://github.com/ggerganov/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
108-
cmake -B build -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL=OFF -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF
108+
cmake -B build -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL=OFF -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF
109109
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
110110
111111
- name: Test
@@ -222,7 +222,7 @@ jobs:
222222
run: |
223223
mkdir build
224224
cd build
225-
cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF
225+
cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF
226226
cmake --build . --config Release -j $(nproc)
227227
228228
- name: Test
@@ -696,22 +696,20 @@ jobs:
696696
strategy:
697697
matrix:
698698
include:
699-
- build: 'rpc-x64'
700-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=ON'
701699
- build: 'noavx-x64'
702-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DBUILD_SHARED_LIBS=ON'
700+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DBUILD_SHARED_LIBS=ON'
703701
- build: 'avx2-x64'
704-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
702+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=ON'
705703
- build: 'avx-x64'
706-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX2=OFF -DBUILD_SHARED_LIBS=ON'
704+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX2=OFF -DBUILD_SHARED_LIBS=ON'
707705
- build: 'avx512-x64'
708-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
706+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
709707
- build: 'openblas-x64'
710-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_BLAS=ON -DBUILD_SHARED_LIBS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
708+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BLAS=ON -DBUILD_SHARED_LIBS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
711709
- build: 'kompute-x64'
712-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON'
710+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON'
713711
- build: 'vulkan-x64'
714-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_VULKAN=ON -DBUILD_SHARED_LIBS=ON'
712+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_VULKAN=ON -DBUILD_SHARED_LIBS=ON'
715713
- build: 'llvm-arm64'
716714
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
717715
- build: 'msvc-arm64'

‎.github/workflows/python-check-requirements.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,13 @@ on:
66
- '.github/workflows/python-check-requirements.yml'
77
- 'scripts/check-requirements.sh'
88
- 'convert*.py'
9-
- 'requirements.txt'
10-
- 'requirements/*.txt'
9+
- '**/requirements*.txt'
1110
pull_request:
1211
paths:
1312
- '.github/workflows/python-check-requirements.yml'
1413
- 'scripts/check-requirements.sh'
1514
- 'convert*.py'
16-
- 'requirements.txt'
17-
- 'requirements/*.txt'
15+
- '**/requirements*.txt'
1816

1917
concurrency:
2018
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,12 @@ Unless otherwise noted these projects are open-source with permissive licensing:
186186

187187
- [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from HuggingFace Hub and convert them to GGML
188188
- [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption
189+
- [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage
189190

190191
**Infrastructure:**
191192

192193
- [Paddler](https://github.com/distantmagic/paddler) - Stateful load balancer custom-tailored for llama.cpp
194+
- [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs
193195

194196
**Games:**
195197
- [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you.

‎common/grammar-parser.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,9 @@ namespace grammar_parser {
369369
}
370370
// Validate the state to ensure that all rules are defined
371371
for (const auto & rule : state.rules) {
372+
if (rule.empty()) {
373+
throw std::runtime_error("Undefined rule");
374+
}
372375
for (const auto & elem : rule) {
373376
if (elem.type == LLAMA_GRETYPE_RULE_REF) {
374377
// Ensure that the rule at that location exists

‎examples/llava/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
--extra-index-url https://download.pytorch.org/whl/cpu
33
pillow~=10.2.0
44
torch~=2.2.1
5-
torchvision==0.17.1
5+
torchvision~=0.17.1

‎examples/server/server.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,7 @@ struct server_context {
631631

632632
bool clean_kv_cache = true;
633633
bool add_bos_token = true;
634+
bool has_eos_token = false;
634635

635636
int32_t n_ctx; // total context for all clients / slots
636637

@@ -693,7 +694,7 @@ struct server_context {
693694
n_ctx = llama_n_ctx(ctx);
694695

695696
add_bos_token = llama_should_add_bos_token(model);
696-
GGML_ASSERT(llama_add_eos_token(model) != 1);
697+
has_eos_token = llama_add_eos_token(model) != 1;
697698

698699
return true;
699700
}
@@ -1031,7 +1032,7 @@ struct server_context {
10311032
{
10321033
slot.sparams.logit_bias.clear();
10331034

1034-
if (json_value(data, "ignore_eos", false)) {
1035+
if (json_value(data, "ignore_eos", false) && has_eos_token) {
10351036
slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
10361037
}
10371038

‎ggml/src/ggml.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21129,7 +21129,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
2112921129
(int64_t) info->ne[2] *
2113021130
(int64_t) info->ne[3];
2113121131

21132-
if (ne % ggml_blck_size(info->type) != 0) {
21132+
if (ggml_blck_size(info->type) == 0 || ne % ggml_blck_size(info->type) != 0) {
2113321133
fprintf(stderr, "%s: tensor '%s' of type %d (%s) number of elements (%" PRId64 ") is not a multiple of block size (%" PRId64 ")\n",
2113421134
__func__, info->name.data, (int) info->type, ggml_type_name(info->type), ne, ggml_blck_size(info->type));
2113521135
fclose(file);

0 commit comments

Comments
 (0)