Skip to content

Commit ffc73fa

Browse files
rutikthakrerutikthakre
committed
feat: add embed components
1 parent 309e2ea commit ffc73fa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+11344
-8
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -381,4 +381,4 @@ jobs:
381381
- name: Login GH CLI
382382
shell: bash
383383
run: gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
384-
- run: gh release upload -R golemcloud/golem-llm --clobber ${{ github.ref_name }} components/release/*.wasm
384+
- run: gh release upload -R golemcloud/golem-llm --clobber ${{ github.ref_name }} components/release/*.wasm

‎Cargo.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ members = [
99
"llm/ollama",
1010
"llm/openai",
1111
"llm/openrouter",
12+
"embed/embed",
13+
"embed/cohere",
14+
"embed/hugging-face",
15+
"embed/openai",
16+
"embed/voyageai",
1217
"websearch/websearch",
1318
"websearch/brave",
1419
"websearch/google",
@@ -49,6 +54,7 @@ golem-search = { path = "search/search", version = "0.0.0", default-features = f
4954
golem-graph = { path = "graph/graph", version = "0.0.0", default-features = false }
5055
golem-video = { path = "video/video", version = "0.0.0", default-features = false }
5156
golem-stt = { path = "stt/stt", version = "0.0.0", default-features = false }
57+
golem-embed = { path = "embed/embed", version = "0.0.0", default-features = false}
5258
futures = "0.3.31"
5359
golem-rust = "=1.5.5"
5460
log = "0.4.27"

‎Makefile.toml‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ skip_core_tasks = true
66
command = "cargo"
77
args = ["clean"]
88

9+
[tasks.clean-all]
10+
script_runner = "@duckscript"
11+
script = '''
12+
exec find . -type d \( -name "target" -o -name "wit-generated" -o -name "golem-temp" \) -exec rm -rf {} +
13+
'''
14+
915
[tasks.unit-tests]
1016
command = "cargo"
1117
args = ["test"]
@@ -14,7 +20,7 @@ args = ["test"]
1420
script_runner = "@duckscript"
1521
script = '''
1622
17-
domains = array llm websearch search video exec graph stt
23+
domains = array llm embed websearch search video exec graph stt
1824
1925
# if there is no domain passed run for every domain
2026
if is_empty ${1}
@@ -30,7 +36,7 @@ end
3036
script_runner = "@duckscript"
3137
script = '''
3238
33-
domains = array llm websearch search video exec graph stt
39+
domains = array llm embed websearch search video exec graph stt
3440
3541
# if there is no domain passed run for every domain
3642
if is_empty ${1}
@@ -46,7 +52,7 @@ end
4652
script_runner = "@duckscript"
4753
script = '''
4854
#!/bin/bash
49-
domains = array llm websearch search video exec graph stt
55+
domains = array llm embed websearch search video exec graph stt
5056
5157
# if there is no domain passed run for every domain
5258
if is_empty ${1}
@@ -62,7 +68,7 @@ end
6268
script_runner = "@duckscript"
6369
script = '''
6470
#!/bin/bash
65-
domains = array llm websearch search exec graph stt
71+
domains = array llm embed websearch search exec graph stt
6672
6773
# if there is no domain passed run for every domain
6874
if is_empty ${1}
@@ -77,7 +83,7 @@ end
7783
[tasks.wit]
7884
script_runner = "@duckscript"
7985
script = '''
80-
domains = array llm websearch search video exec graph stt
86+
domains = array llm embed websearch search video exec graph stt
8187
8288
# if there is no domain passed run for every domain
8389
if is_empty ${1}
@@ -93,7 +99,7 @@ end
9399
description = "Builds all test components with golem-cli"
94100
script_runner = "@duckscript"
95101
script = '''
96-
domains = array llm websearch search video exec graph stt
102+
domains = array llm embed websearch search video exec graph stt
97103
98104
# if there is no domain passed run for every domain
99105
if is_empty ${1}
@@ -188,6 +194,26 @@ else
188194
cp target/debug/golem_exec_javascript.wasm components/debug/golem_exec_javascript.wasm
189195
end
190196
end
197+
198+
199+
200+
embed_targets = array embed_openai embed_cohere embed_hugging_face embed_voyageai
201+
202+
for target in ${embed_targets}
203+
if ${is_portable}
204+
if ${is_dev}
205+
cp target/wasm32-wasip1/debug/golem_${target}.wasm components/debug/golem_${target}-portable-dev.wasm
206+
else
207+
cp target/wasm32-wasip1/debug/golem_${target}.wasm components/debug/golem_${target}-portable.wasm
208+
end
209+
else
210+
if ${is_dev}
211+
cp target/wasm32-wasip1/debug/golem_${target}.wasm components/debug/golem_${target}-dev.wasm
212+
else
213+
cp target/wasm32-wasip1/debug/golem_${target}.wasm components/debug/golem_${target}.wasm
214+
end
215+
end
216+
end
191217
'''
192218

193219
[tasks.copy-release-artifacts]
@@ -200,7 +226,7 @@ environment = get_env "ENVIRONMENT"
200226
is_dev = eq ${environment} "dev"
201227
202228
203-
targets = array llm_openai llm_anthropic llm_grok llm_openrouter llm_ollama web_search_brave web_search_google web_search_serper web_search_tavily search_algolia search_elasticsearch search_meilisearch search_opensearch search_typesense video_kling video_runway video_veo video_stability graph_arangodb graph_janusgraph graph_neo4j stt_aws stt_azure stt_deepgram stt_google stt_whisper
229+
targets = array llm_openai llm_anthropic llm_grok llm_openrouter llm_ollama web_search_brave web_search_google web_search_serper web_search_tavily search_algolia search_elasticsearch search_meilisearch search_opensearch search_typesense video_kling video_runway video_veo video_stability graph_arangodb graph_janusgraph graph_neo4j stt_aws stt_azure stt_deepgram stt_google stt_whisper
204230
205231
for target in ${targets}
206232
echo "Copying artifacts for ${target}..."
@@ -241,6 +267,25 @@ else
241267
cp target/release/golem_exec_javascript.wasm components/release/golem_exec_javascript.wasm
242268
end
243269
end
270+
271+
272+
embed_targets = array embed_openai embed_cohere embed_hugging_face embed_voyageai
273+
274+
for target in ${embed_targets}
275+
if ${is_portable}
276+
if ${is_dev}
277+
cp target/wasm32-wasip1/release/golem_${target}.wasm components/release/golem_${target}-portable-dev.wasm
278+
else
279+
cp target/wasm32-wasip1/release/golem_${target}.wasm components/release/golem_${target}-portable.wasm
280+
end
281+
else
282+
if ${is_dev}
283+
cp target/wasm32-wasip1/release/golem_${target}.wasm components/release/golem_${target}-dev.wasm
284+
else
285+
cp target/wasm32-wasip1/release/golem_${target}.wasm components/release/golem_${target}.wasm
286+
end
287+
end
288+
end
244289
'''
245290

246291
# Maintenance tasks

‎embed/Makefile.toml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
[config]
2+
default_to_workspace = false
3+
skip_core_tasks = true
4+
5+
[tasks.build]
6+
run_task = { name = [
7+
"build-openai",
8+
"build-cohere",
9+
"build-hugging-face",
10+
"build-voyageai",
11+
] }
12+
13+
[tasks.build-portable]
14+
run_task = { name = [
15+
"build-openai-portable",
16+
"build-cohere-portable",
17+
"build-hugging-face-portable",
18+
"build-voyageai-portable",
19+
] }
20+
21+
[tasks.release-build]
22+
run_task = { name = [
23+
"release-build-openai",
24+
"release-build-cohere",
25+
"release-build-hugging-face",
26+
"release-build-voyageai",
27+
] }
28+
29+
[tasks.release-build-portable]
30+
run_task = { name = [
31+
"release-build-openai-portable",
32+
"release-build-cohere-portable",
33+
"release-build-hugging-face-portable",
34+
"release-build-voyageai-portable",
35+
] }
36+
37+
[tasks.build-openai]
38+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
39+
command = "cargo-component"
40+
args = ["build", "-p", "golem-embed-openai"]
41+
42+
[tasks.build-openai-portable]
43+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
44+
command = "cargo-component"
45+
args = ["build", "-p", "golem-embed-openai", "--no-default-features"]
46+
47+
[tasks.build-cohere]
48+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
49+
command = "cargo-component"
50+
args = ["build", "-p", "golem-embed-cohere"]
51+
52+
[tasks.build-cohere-portable]
53+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
54+
command = "cargo-component"
55+
args = ["build", "-p", "golem-embed-cohere", "--no-default-features"]
56+
57+
[tasks.build-hugging-face]
58+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
59+
command = "cargo-component"
60+
args = ["build", "-p", "golem-embed-hugging-face"]
61+
62+
[tasks.build-hugging-face-portable]
63+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
64+
command = "cargo-component"
65+
args = ["build", "-p", "golem-embed-hugging-face", "--no-default-features"]
66+
67+
[tasks.build-voyageai]
68+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
69+
command = "cargo-component"
70+
args = ["build", "-p", "golem-embed-voyageai"]
71+
72+
[tasks.build-voyageai-portable]
73+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
74+
command = "cargo-component"
75+
args = ["build", "-p", "golem-embed-voyageai", "--no-default-features"]
76+
77+
[tasks.release-build-openai]
78+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
79+
command = "cargo-component"
80+
args = ["build", "-p", "golem-embed-openai", "--release"]
81+
82+
[tasks.release-build-openai-portable]
83+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
84+
command = "cargo-component"
85+
args = ["build", "-p", "golem-embed-openai", "--release", "--no-default-features"]
86+
87+
[tasks.release-build-cohere]
88+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
89+
command = "cargo-component"
90+
args = ["build", "-p", "golem-embed-cohere", "--release"]
91+
92+
[tasks.release-build-cohere-portable]
93+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
94+
command = "cargo-component"
95+
args = ["build", "-p", "golem-embed-cohere", "--release", "--no-default-features"]
96+
97+
[tasks.release-build-hugging-face]
98+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
99+
command = "cargo-component"
100+
args = ["build", "-p", "golem-embed-hugging-face", "--release"]
101+
102+
[tasks.release-build-hugging-face-portable]
103+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
104+
command = "cargo-component"
105+
args = ["build", "-p", "golem-embed-hugging-face", "--release", "--no-default-features"]
106+
107+
[tasks.release-build-voyageai]
108+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
109+
command = "cargo-component"
110+
args = ["build", "-p", "golem-embed-voyageai", "--release"]
111+
112+
[tasks.release-build-voyageai-portable]
113+
install_crate = { crate_name = "cargo-component", version = "0.20.0" }
114+
command = "cargo-component"
115+
args = ["build", "-p", "golem-embed-voyageai", "--release", "--no-default-features"]
116+
117+
[tasks.wit-update]
118+
install_crate = { crate_name = "wit-deps-cli" }
119+
command = "wit-deps"
120+
args = ["update"]
121+
122+
[tasks.wit]
123+
dependencies = ["wit-update"]
124+
125+
script_runner = "@duckscript"
126+
script = """
127+
modules = array embed openai cohere hugging-face voyageai ../test/embed
128+
129+
for module in ${modules}
130+
rm -r ${module}/wit/deps
131+
mkdir ${module}/wit/deps/golem-embed
132+
cp wit/golem-embed.wit ${module}/wit/deps/golem-embed/golem-embed.wit
133+
cp wit/deps/wasi:io ${module}/wit/deps
134+
135+
echo "Copied WIT for module embed::${module}"
136+
end
137+
138+
"""
139+
140+
[tasks.build-test-components]
141+
dependencies = ["build"]
142+
install_crate = "cargo-binstall"
143+
description = "Builds embed test components with golem-cli"
144+
script = '''
145+
cargo-binstall [email protected] --locked --no-confirm
146+
cargo-binstall wac-cli --locked --no-confirm
147+
cd ../test/embed
148+
149+
golem-cli --version
150+
golem-cli app clean
151+
golem-cli app build -b openai-debug
152+
golem-cli app clean
153+
golem-cli app build -b cohere-debug
154+
golem-cli app clean
155+
golem-cli app build -b hugging-face-debug
156+
golem-cli app clean
157+
golem-cli app build -b voyageai-debug
158+
'''
159+
160+
161+

0 commit comments

Comments
 (0)