Skip to content

Commit 581f64d

Browse files
asg017asg017
committed
rename to embedfile
1 parent 92065a5 commit 581f64d

23 files changed

+160
-113
lines changed

‎TODO‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
```
22
./make -j8
3-
./make o//llama.cpp/embedr/embedr
4-
./o/llama.cpp/embedr/embedr --version
5-
./o/llama.cpp/embedr/embedr
3+
./make o//llama.cpp/embedfile/embedfile
4+
./o/llama.cpp/embedfile/embedfile --version
5+
./o/llama.cpp/embedfile/embedfile
66
```
77

88

99
```
10-
EMBEDR_MODEL_PATH=$PWD/models/mxbai-embed-xsmall-v1-f16.gguf ./o/llama.cpp/embedr/embedr backfill tmp.smol.db nyt_headlines headline
10+
EMBEDFILE_MODEL_PATH=$PWD/models/mxbai-embed-xsmall-v1-f16.gguf ./o/llama.cpp/embedfile/embedfile backfill tmp.smol.db nyt_headlines headline
1111

12-
./o/llama.cpp/embedr/embedr backfill tmp.smol.db nyt_headlines headline
12+
./o/llama.cpp/embedfile/embedfile backfill tmp.smol.db nyt_headlines headline
1313
```

‎embedfile.mk‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
prefix=dist
2+
3+
$(prefix):
4+
mkdir -p $@
5+
echo "*" > $(prefix)/.gitignore
6+
7+
MODELS_DIR=$(prefix)/.models
8+
9+
$(MODELS_DIR): $(prefix)
10+
mkdir -p $@
11+
12+
.PHONY: models all
13+
14+
EMBEDFILE=./o/llama.cpp/embedfile/embedfile
15+
16+
MODEL_MXBAI=mxbai-embed-xsmall-v1-f16
17+
MODEL_SNOWFLAKE=snowflake-arctic-embed-m-v1.5-f16
18+
MODEL_NOMIC=nomic-embed-text-v1.5.f16
19+
MODEL_ALLMINI=all-MiniLM-L6-v2.f16
20+
21+
$(MODELS_DIR)/$(MODEL_MXBAI).gguf: $(MODELS_DIR)
22+
curl -L -o $@ 'https://huggingface.co/mixedbread-ai/mxbai-embed-xsmall-v1/resolve/main/gguf/mxbai-embed-xsmall-v1-f16.gguf'
23+
24+
$(MODELS_DIR)/$(MODEL_SNOWFLAKE).gguf: $(MODELS_DIR)
25+
curl -L -o $@ 'https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5/resolve/main/gguf/snowflake-arctic-embed-m-v1.5-f16.gguf'
26+
27+
$(MODELS_DIR)/$(MODEL_NOMIC).gguf: $(MODELS_DIR)
28+
curl -L -o $@ 'https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf'
29+
30+
$(MODELS_DIR)/$(MODEL_ALLMINI).gguf: $(MODELS_DIR)
31+
curl -L -o $@ 'https://huggingface.co/asg017/sqlite-lembed-model-examples/resolve/main/all-MiniLM-L6-v2/all-MiniLM-L6-v2.e4ce9877.f16.gguf'
32+
33+
models: \
34+
$(MODELS_DIR)/$(MODEL_MXBAI).gguf \
35+
$(MODELS_DIR)/$(MODEL_SNOWFLAKE).gguf \
36+
$(MODELS_DIR)/$(MODEL_NOMIC).gguf \
37+
$(MODELS_DIR)/$(MODEL_ALLMINI).gguf
38+
39+
dist/$(MODEL_MXBAI).embedfile: $(MODELS_DIR)/$(MODEL_MXBAI).gguf $(EMBEDFILE) embedfile.mk
40+
cp $(EMBEDFILE) $@
41+
echo "-m\n$(MODEL_MXBAI).gguf\n..." > .args
42+
./o/llamafile/zipalign -j0 $@ $< .args
43+
rm .args
44+
45+
dist/$(MODEL_SNOWFLAKE).embedfile: $(MODELS_DIR)/$(MODEL_SNOWFLAKE).gguf $(EMBEDFILE) embedfile.mk
46+
cp $(EMBEDFILE) $@
47+
echo "-m\n$(MODEL_SNOWFLAKE).gguf\n..." > .args
48+
./o/llamafile/zipalign -j0 $@ $< .args
49+
rm .args
50+
51+
dist/$(MODEL_NOMIC).embedfile: $(MODELS_DIR)/$(MODEL_NOMIC).gguf $(EMBEDFILE) embedfile.mk
52+
cp $(EMBEDFILE) $@
53+
echo "-m\n$(MODEL_NOMIC).gguf\n..." > .args
54+
./o/llamafile/zipalign -j0 $@ $< .args
55+
rm .args
56+
57+
dist/$(MODEL_ALLMINI).embedfile: $(MODELS_DIR)/$(MODEL_ALLMINI).gguf $(EMBEDFILE) embedfile.mk
58+
cp $(EMBEDFILE) $@
59+
echo "-m\n$(MODEL_ALLMINI).gguf\n..." > .args
60+
./o/llamafile/zipalign -j0 $@ $< .args
61+
rm .args
62+
63+
all: \
64+
dist/$(MODEL_MXBAI).embedfile \
65+
dist/$(MODEL_SNOWFLAKE).embedfile \
66+
dist/$(MODEL_NOMIC).embedfile \
67+
dist/$(MODEL_ALLMINI).embedfile

‎embedr.mk‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎llama.cpp/BUILD.mk‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ include llama.cpp/server/BUILD.mk
2626
include llama.cpp/main/BUILD.mk
2727
include llama.cpp/imatrix/BUILD.mk
2828
include llama.cpp/quantize/BUILD.mk
29-
include llama.cpp/embedr/BUILD.mk
29+
include llama.cpp/embedfile/BUILD.mk
3030
include llama.cpp/perplexity/BUILD.mk
3131
include llama.cpp/llama-bench/BUILD.mk
3232

@@ -90,7 +90,7 @@ $(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk
9090
o/$(MODE)/llama.cpp: \
9191
o/$(MODE)/llama.cpp/main \
9292
o/$(MODE)/llama.cpp/llava \
93-
o/$(MODE)/llama.cpp/embedr \
93+
o/$(MODE)/llama.cpp/embedfile \
9494
o/$(MODE)/llama.cpp/server \
9595
o/$(MODE)/llama.cpp/imatrix \
9696
o/$(MODE)/llama.cpp/quantize \

‎llama.cpp/embedfile/BUILD.mk‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐
2+
#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘
3+
4+
PKGS += LLAMA_CPP_EMBEDFILE
5+
6+
LLAMA_CPP_EMBEDFILE_FILES := $(wildcard llama.cpp/embedfile/*)
7+
LLAMA_CPP_EMBEDFILE_HDRS = $(filter %.h,$(LLAMA_CPP_EMBEDFILE_FILES))
8+
LLAMA_CPP_EMBEDFILE_SRCS_C = $(filter %.c,$(LLAMA_CPP_EMBEDFILE_FILES))
9+
LLAMA_CPP_EMBEDFILE_SRCS_CPP = $(filter %.cpp,$(LLAMA_CPP_EMBEDFILE_FILES))
10+
LLAMA_CPP_EMBEDFILE_SRCS = $(LLAMA_CPP_EMBEDFILE_SRCS_C) $(LLAMA_CPP_EMBEDFILE_SRCS_CPP)
11+
12+
LLAMA_CPP_EMBEDFILE_OBJS = \
13+
$(LLAMA_CPP_EMBEDFILE_SRCS_C:%.c=o/$(MODE)/%.o) \
14+
$(LLAMA_CPP_EMBEDFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o)
15+
16+
17+
o/$(MODE)/llama.cpp/embedfile/embedfile.a: $(LLAMA_CPP_EMBEDFILE_SRCS_C)
18+
19+
o/$(MODE)/llama.cpp/embedfile/sqlite3.o: llama.cpp/embedfile/sqlite3.c
20+
o/$(MODE)/llama.cpp/embedfile/sqlite3.a: o/$(MODE)/llama.cpp/embedfile/sqlite3.o
21+
22+
o/$(MODE)/llama.cpp/embedfile/sqlite-vec.o: llama.cpp/embedfile/sqlite-vec.c
23+
o/$(MODE)/llama.cpp/embedfile/sqlite-vec.a: o/$(MODE)/llama.cpp/embedfile/sqlite-vec.o
24+
25+
o/$(MODE)/llama.cpp/embedfile/sqlite-csv.o: llama.cpp/embedfile/sqlite-csv.c
26+
o/$(MODE)/llama.cpp/embedfile/sqlite-csv.a: o/$(MODE)/llama.cpp/embedfile/sqlite-csv.o
27+
28+
o/$(MODE)/llama.cpp/embedfile/sqlite-lines.o: llama.cpp/embedfile/sqlite-lines.c
29+
o/$(MODE)/llama.cpp/embedfile/sqlite-lines.a: o/$(MODE)/llama.cpp/embedfile/sqlite-lines.o
30+
31+
o/$(MODE)/llama.cpp/embedfile/sqlite-lembed.o: llama.cpp/embedfile/sqlite-lembed.c
32+
o/$(MODE)/llama.cpp/embedfile/sqlite-lembed.a: o/$(MODE)/llama.cpp/embedfile/sqlite-lembed.o o/$(MODE)/llama.cpp/llama.cpp.a
33+
34+
o/$(MODE)/llama.cpp/embedfile/shell.o: llama.cpp/embedfile/shell.c
35+
o/$(MODE)/llama.cpp/embedfile/shell.a: o/$(MODE)/llama.cpp/embedfile/shell.o
36+
37+
#o/$(MODE)/llama.cpp/embedfile/embedfile.a: $(LLAMA_CPP_EMBEDFILE_OBJS)
38+
39+
#o/$(MODE)/llama.cpp/embedfile/sqlite3.o: private COPTS += -O3
40+
41+
o/$(MODE)/llama.cpp/embedfile/embedfile: \
42+
o/$(MODE)/llama.cpp/embedfile/shell.a \
43+
o/$(MODE)/llama.cpp/embedfile/embedfile.o \
44+
o/$(MODE)/llama.cpp/embedfile/embedfile.1.asc.zip.o \
45+
o/$(MODE)/llama.cpp/llama.cpp.a \
46+
o/$(MODE)/llama.cpp/embedfile/sqlite3.a \
47+
o/$(MODE)/llama.cpp/embedfile/sqlite-csv.a \
48+
o/$(MODE)/llama.cpp/embedfile/sqlite-vec.a \
49+
o/$(MODE)/llama.cpp/embedfile/sqlite-lines.a \
50+
o/$(MODE)/llama.cpp/embedfile/sqlite-lembed.a
51+
52+
$(LLAMA_CPP_EMBEDFILE_OBJS): private CCFLAGS += -DSQLITE_CORE
53+
54+
.PHONY: o/$(MODE)/llama.cpp/embedfile
55+
o/$(MODE)/llama.cpp/embedfile: \
56+
o/$(MODE)/llama.cpp/embedfile/embedfile
57+
58+
$(LLAMA_CPP_EMBEDFILE_OBJS): llama.cpp/BUILD.mk llama.cpp/embedfile/BUILD.mk

‎llama.cpp/embedr/embedr.1‎ renamed to ‎llama.cpp/embedfile/embedfile.1‎

File renamed without changes.

‎llama.cpp/embedr/embedr.1.asc‎ renamed to ‎llama.cpp/embedfile/embedfile.1.asc‎

File renamed without changes.

‎llama.cpp/embedr/embedr.c‎ renamed to ‎llama.cpp/embedfile/embedfile.c‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22
// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
33
#include "llama.cpp/llama.h"
44
#include "llamafile/version.h"
5-
#include "llama.cpp/embedr/embedr.h"
6-
#include "llama.cpp/embedr/sqlite3.h"
7-
#include "llama.cpp/embedr/sqlite-vec.h"
8-
#include "llama.cpp/embedr/sqlite-lembed.h"
9-
#include "llama.cpp/embedr/sqlite-csv.h"
10-
#include "llama.cpp/embedr/sqlite-lines.h"
11-
#include "llama.cpp/embedr/shell.h"
5+
#include "llama.cpp/embedfile/embedfile.h"
6+
#include "llama.cpp/embedfile/sqlite3.h"
7+
#include "llama.cpp/embedfile/sqlite-vec.h"
8+
#include "llama.cpp/embedfile/sqlite-lembed.h"
9+
#include "llama.cpp/embedfile/sqlite-csv.h"
10+
#include "llama.cpp/embedfile/sqlite-lines.h"
11+
#include "llama.cpp/embedfile/shell.h"
1212
#include <string.h>
1313

1414
#include <stdlib.h>
@@ -23,22 +23,22 @@ int64_t time_ms(void) {
2323
return (int64_t)ts.tv_sec*1000 + (int64_t)ts.tv_nsec/1000000;
2424
}
2525

26-
char * EMBEDR_MODEL = NULL;
26+
char * EMBEDFILE_MODEL = NULL;
2727

28-
void embedr_version(sqlite3_context * context, int argc, sqlite3_value **value) {
29-
sqlite3_result_text(context, EMBEDR_VERSION, -1, SQLITE_STATIC);
28+
void embedfile_version(sqlite3_context * context, int argc, sqlite3_value **value) {
29+
sqlite3_result_text(context, EMBEDFILE_VERSION, -1, SQLITE_STATIC);
3030
}
3131

32-
int embedr_sqlite3_init(sqlite3 * db) {
32+
int embedfile_sqlite3_init(sqlite3 * db) {
3333
int rc;
3434

3535
rc = sqlite3_vec_init(db, NULL, NULL); assert(rc == SQLITE_OK);
3636
rc = sqlite3_lembed_init(db, NULL, NULL); assert(rc == SQLITE_OK);
3737
rc = sqlite3_csv_init(db, NULL, NULL); assert(rc == SQLITE_OK);
3838
rc = sqlite3_lines_init(db, NULL, NULL); assert(rc == SQLITE_OK);
39-
rc = sqlite3_create_function_v2(db, "embedr_version",0, SQLITE_DETERMINISTIC | SQLITE_UTF8, NULL, embedr_version, NULL, NULL, NULL); assert(rc == SQLITE_OK);
39+
rc = sqlite3_create_function_v2(db, "embedfile_version",0, SQLITE_DETERMINISTIC | SQLITE_UTF8, NULL, embedfile_version, NULL, NULL, NULL); assert(rc == SQLITE_OK);
4040

41-
if(!EMBEDR_MODEL) {
41+
if(!EMBEDFILE_MODEL) {
4242
return SQLITE_OK;
4343
}
4444
sqlite3_stmt * stmt;
@@ -47,7 +47,7 @@ int embedr_sqlite3_init(sqlite3 * db) {
4747
assert(rc == SQLITE_OK);
4848
return rc;
4949
}
50-
sqlite3_bind_text(stmt, 1, EMBEDR_MODEL, -1, SQLITE_STATIC);
50+
sqlite3_bind_text(stmt, 1, EMBEDFILE_MODEL, -1, SQLITE_STATIC);
5151
sqlite3_step(stmt);
5252
rc = sqlite3_finalize(stmt);
5353
assert(rc == SQLITE_OK);
@@ -122,7 +122,7 @@ int cmd_index(char * filename, char * target_column) {
122122
rc = sqlite3_exec(db, "PRAGMA page_size=16384;", NULL, NULL, NULL);
123123
assert(rc == SQLITE_OK);
124124

125-
rc = embedr_sqlite3_init(db);
125+
rc = embedfile_sqlite3_init(db);
126126
assert(rc == SQLITE_OK);
127127

128128
if(sqlite3_strlike("%.csv", filename, 0) == 0) {
@@ -238,7 +238,7 @@ int cmd_backfill(char * dbPath, char * table, char * column) {
238238
return rc;
239239
}
240240

241-
rc = embedr_sqlite3_init(db);
241+
rc = embedfile_sqlite3_init(db);
242242
assert(rc == SQLITE_OK);
243243

244244
rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL);
@@ -353,7 +353,7 @@ int cmd_embed(char * source) {
353353
rc = sqlite3_open(":memory:", &db);
354354
assert(rc == SQLITE_OK);
355355

356-
rc = embedr_sqlite3_init(db);
356+
rc = embedfile_sqlite3_init(db);
357357
assert(rc == SQLITE_OK);
358358

359359
rc = sqlite3_prepare_v2(db, "select vec_to_json(lembed(?))", -1, &stmt, NULL);
@@ -390,12 +390,12 @@ int main(int argc, char ** argv) {
390390
char * arg = argv[i];
391391
if(sqlite3_stricmp(arg, "--model") == 0 || sqlite3_stricmp(arg, "-m") == 0) {
392392
assert(++i <= argc);
393-
EMBEDR_MODEL = argv[i];
393+
EMBEDFILE_MODEL = argv[i];
394394
}
395395
else if(sqlite3_stricmp(arg, "--version") == 0 || sqlite3_stricmp(arg, "-v") == 0) {
396396
fprintf(stderr,
397-
"embedr %s, llamafile %s, SQLite %s, sqlite-vec=%s, sqlite-lembed=%s\n",
398-
EMBEDR_VERSION,
397+
"embedfile %s, llamafile %s, SQLite %s, sqlite-vec=%s, sqlite-lembed=%s\n",
398+
EMBEDFILE_VERSION,
399399
LLAMAFILE_VERSION_STRING,
400400
sqlite3_version,
401401
SQLITE_VEC_VERSION,

‎llama.cpp/embedfile/embedfile.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#ifndef EMBEDFILE_H
2+
#define EMBEDFILE_H
3+
4+
#define EMBEDFILE_VERSION "0.0.1-alpha.1"
5+
6+
#endif

‎llama.cpp/embedr/shell.c‎ renamed to ‎llama.cpp/embedfile/shell.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25607,8 +25607,8 @@ static void open_db(ShellState *p, int openFlags){
2560725607
sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0,
2560825608
editFunc, 0, 0);
2560925609
#endif
25610-
extern int embedr_sqlite3_init(sqlite3 *);
25611-
embedr_sqlite3_init(p->db);
25610+
extern int embedfile_sqlite3_init(sqlite3 *);
25611+
embedfile_sqlite3_init(p->db);
2561225612

2561325613
if( p->openMode==SHELL_OPEN_ZIPFILE ){
2561425614
char *zSql = sqlite3_mprintf(

0 commit comments

Comments
 (0)