Skip to content

Commit bde85cd

Browse files
committed
[aux] Memory example (simple)
Adapt simple example to showcase how to load from memory. Can be configured with environment variables. Qwen3, for example, can be used with the simple example.
1 parent 28751d1 commit bde85cd

File tree

2 files changed

+8
-9
lines changed

2 files changed

+8
-9
lines changed

examples/simple/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
set(TARGET llama-simple)
22
add_executable(${TARGET} simple.cpp)
33
install(TARGETS ${TARGET} RUNTIME)
4-
target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT})
4+
target_link_libraries(${TARGET} PRIVATE llama llama-common-test ${CMAKE_THREAD_LIBS_INIT})
55
target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/simple/simple.cpp

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1-
#include "llama.h"
1+
#include "llama-cpp.h"
22
#include <cstdio>
33
#include <cstring>
44
#include <string>
5-
#include <vector>
65

76
static void print_usage(int, char ** argv) {
87
printf("\nexample usage:\n");
98
printf("\n %s -m model.gguf [-n n_predict] [-ngl n_gpu_layers] [prompt]\n", argv[0]);
9+
printf("\n Optional environment variables: LLAMA_EXAMPLE_MEMORY_BUFFER LLAMA_EXAMPLE_MEMORY_BUFFER_SPLIT");
1010
printf("\n");
1111
}
1212

13+
#include "load_into_memory.h"
14+
1315
int main(int argc, char ** argv) {
1416
// path to the model gguf file
1517
std::string model_path;
@@ -83,12 +85,9 @@ int main(int argc, char ** argv) {
8385
llama_model_params model_params = llama_model_default_params();
8486
model_params.n_gpu_layers = ngl;
8587

86-
llama_model * model = llama_model_load_from_file(model_path.c_str(), model_params);
87-
88-
if (model == NULL) {
89-
fprintf(stderr , "%s: error: unable to load model\n" , __func__);
90-
return 1;
91-
}
88+
llama_model * model = memory_configuration_env_is_set() ?
89+
load_model_from_memory_configuration(model_path.c_str(), model_params) :
90+
llama_model_load_from_file(model_path.c_str(), model_params);
9291

9392
const llama_vocab * vocab = llama_model_get_vocab(model);
9493
// tokenize the prompt

0 commit comments

Comments
 (0)