Skip to content

feat: support local ai embedding, local ai search, local document content search #7839

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
appflowy merged 35 commits into from
May 2, 2025
Merged

feat: support local ai embedding, local ai search, local document content search #7839

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
6e71dda
chore: generate embeddings
appflowy Apr 26, 2025
f589d45
chore: save embedding
appflowy Apr 27, 2025
6d79c67
chore: vec sqlite
appflowy Apr 27, 2025
a641171
chore: clippy
appflowy Apr 27, 2025
fff9c77
chore: init vector
appflowy Apr 27, 2025
14d9830
chore: create vector db
appflowy Apr 27, 2025
6027a99
chore: periodically write embedding
appflowy Apr 28, 2025
3fbe711
chore: fix compile
appflowy Apr 28, 2025
f695137
chore: skip write
appflowy Apr 28, 2025
d6187ba
chore: impl search
appflowy Apr 28, 2025
f35ae04
Merge branch 'main' into local_embedding
appflowy Apr 28, 2025
d913716
fix: test
appflowy Apr 28, 2025
1148928
fix: stop scheduler
appflowy Apr 28, 2025
d3a1511
fix: search
appflowy Apr 28, 2025
840c213
chore: add test
appflowy Apr 28, 2025
4786787
chore: update schema
appflowy Apr 28, 2025
5345660
chore: index all
appflowy Apr 29, 2025
881300b
chore: index
appflowy Apr 29, 2025
775dce2
chore: search document content
appflowy Apr 30, 2025
65039a4
chore: index document content and title
appflowy Apr 30, 2025
78a2d10
chore: index document content and title
appflowy Apr 30, 2025
a623909
chore: index all view
appflowy Apr 30, 2025
3a77d02
chore: rename trait
appflowy Apr 30, 2025
930ab2f
refactor: remove folder index manager
appflowy May 1, 2025
af066b0
Merge branch 'main' into local_embedding
appflowy May 1, 2025
e738559
chore: index folder changes
appflowy May 1, 2025
54891d7
chore: delete folder search
appflowy May 1, 2025
98eeff7
chore: update logs
appflowy May 1, 2025
869a66c
chore: update logs
appflowy May 1, 2025
b678341
chore: search
appflowy May 1, 2025
c435e44
chore: add search test
appflowy May 1, 2025
8860281
chore: fmt
appflowy May 1, 2025
aa057dc
chore: fmt
appflowy May 1, 2025
86adede
chore: fix test
appflowy May 1, 2025
c9caeb8
chore: fix ios build
appflowy May 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 35 additions & 34 deletions frontend/Makefile.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
#https://github.com/sagiegurari/cargo-make

extend = [
{ path = "scripts/makefile/desktop.toml" },
{ path = "scripts/makefile/mobile.toml" },
{ path = "scripts/makefile/protobuf.toml" },
{ path = "scripts/makefile/tests.toml" },
{ path = "scripts/makefile/docker.toml" },
{ path = "scripts/makefile/env.toml" },
{ path = "scripts/makefile/flutter.toml" },
{ path = "scripts/makefile/tool.toml" },
{ path = "scripts/makefile/tauri.toml" },
{ path = "scripts/makefile/web.toml" },
{ path = "scripts/makefile/desktop.toml" },
{ path = "scripts/makefile/mobile.toml" },
{ path = "scripts/makefile/protobuf.toml" },
{ path = "scripts/makefile/tests.toml" },
{ path = "scripts/makefile/docker.toml" },
{ path = "scripts/makefile/env.toml" },
{ path = "scripts/makefile/flutter.toml" },
{ path = "scripts/makefile/tool.toml" },
{ path = "scripts/makefile/tauri.toml" },
{ path = "scripts/makefile/web.toml" },
]

[config]
Expand Down Expand Up @@ -228,30 +228,31 @@ script = ['''
echo APP_ENVIRONMENT: ${APP_ENVIRONMENT}
echo BUILD_ARCHS: ${BUILD_ARCHS}
echo BUILD_VERSION: ${BUILD_VERSION}
echo RUST_VERSION: "$(rustc --version)"
''']
script_runner = "@shell"

[tasks.setup-crate-type]
private = true
script = [
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} "staticlib" ${CRATE_TYPE}
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} "staticlib" ${CRATE_TYPE}
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
]
script_runner = "@duckscript"

[tasks.restore-crate-type]
private = true
script = [
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} ${CRATE_TYPE} "staticlib"
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} ${CRATE_TYPE} "staticlib"
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
]
script_runner = "@duckscript"

Expand Down Expand Up @@ -279,24 +280,24 @@ TEST_COMPILE_TARGET = "x86_64-pc-windows-msvc"
[tasks.setup-test-crate-type]
private = true
script = [
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} "staticlib" ${TEST_CRATE_TYPE}
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} "staticlib" ${TEST_CRATE_TYPE}
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
]
script_runner = "@duckscript"

[tasks.restore-test-crate-type]
private = true
script = [
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} ${TEST_CRATE_TYPE} "staticlib"
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
"""
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
val = replace ${toml} ${TEST_CRATE_TYPE} "staticlib"
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
assert ${result}
""",
]
script_runner = "@duckscript"

Expand Down
46 changes: 23 additions & 23 deletions frontend/appflowy_flutter/macos/Podfile.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,34 +144,34 @@ EXTERNAL SOURCES:
:path: Flutter/ephemeral/.symlinks/plugins/window_manager/macos

SPEC CHECKSUMS:
app_links: 9028728e32c83a0831d9db8cf91c526d16cc5468
appflowy_backend: 464aeb3e5c6966a41641a2111e5ead72ce2695f7
auto_updater_macos: 3a42f1a06be6981f1a18be37e6e7bf86aa732118
bitsdojo_window_macos: 7959fb0ca65a3ccda30095c181ecb856fae48ea9
connectivity_plus: e74b9f74717d2d99d45751750e266e55912baeb5
desktop_drop: e0b672a7d84c0a6cbc378595e82cdb15f2970a43
device_info_plus: 4fb280989f669696856f8b129e4a5e3cd6c48f76
file_selector_macos: 6280b52b459ae6c590af5d78fc35c7267a3c4b31
flowy_infra_ui: 8760ff42a789de40bf5007a5f176b454722a341e
app_links: 10e0a0ab602ffaf34d142cd4862f29d34b303b2a
appflowy_backend: 865496343de667fc8c600e04b9fd05234e130cf9
auto_updater_macos: 3e3462c418fe4e731917eacd8d28eef7af84086d
bitsdojo_window_macos: 44e3b8fe3dd463820e0321f6256c5b1c16bb6a00
connectivity_plus: 18d3c32514c886e046de60e9c13895109866c747
desktop_drop: 69eeff437544aa619c8db7f4481b3a65f7696898
device_info_plus: 1b14eed9bf95428983aed283a8d51cce3d8c4215
file_selector_macos: cc3858c981fe6889f364731200d6232dac1d812d
flowy_infra_ui: 03301a39ad118771adbf051a664265c61c507f38
FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24
HotKey: 400beb7caa29054ea8d864c96f5ba7e5b4852277
hotkey_manager: b443f35f4d772162937aa73fd8995e579f8ac4e2
irondash_engine_context: 893c7d96d20ce361d7e996f39d360c4c2f9869ba
local_notifier: ebf072651e35ae5e47280ad52e2707375cb2ae4e
package_info_plus: f0052d280d17aa382b932f399edf32507174e870
path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564
hotkey_manager: c32bf0bfe8f934b7bc17ab4ad5c4c142960b023c
irondash_engine_context: da62996ee25616d2f01bbeb85dc115d813359478
local_notifier: e9506bc66fc70311e8bc7291fb70f743c081e4ff
package_info_plus: 12f1c5c2cfe8727ca46cbd0b26677728972d9a5b
path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46
ReachabilitySwift: 32793e867593cfc1177f5d16491e3a197d2fccda
screen_retriever_macos: 452e51764a9e1cdb74b3c541238795849f21557f
screen_retriever_macos: 776e0fa5d42c6163d2bf772d22478df4b302b161
Sentry: da60d980b197a46db0b35ea12cb8f39af48d8854
sentry_flutter: 27892878729f42701297c628eb90e7c6529f3684
share_plus: 510bf0af1a42cd602274b4629920c9649c52f4cc
shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7
sentry_flutter: 2df8b0aab7e4aba81261c230cbea31c82a62dd1b
share_plus: 1fa619de8392a4398bfaf176d441853922614e89
shared_preferences_foundation: fcdcbc04712aee1108ac7fda236f363274528f78
Sparkle: 9c328bdcfbcaf8f030c4b678eadfd0fcb03822d8
sqflite_darwin: 20b2a3a3b70e43edae938624ce550a3cbf66a3d0
super_native_extensions: c2795d6d9aedf4a79fae25cb6160b71b50549189
url_launcher_macos: 0fba8ddabfc33ce0a9afe7c5fef5aab3d8d2d673
webview_flutter_wkwebview: 44d4dee7d7056d5ad185d25b38404436d56c547c
window_manager: e8d0b1431ab6c454f2b5c9ae26004bbfa43469aa
sqflite_darwin: 5a7236e3b501866c1c9befc6771dfd73ffb8702d
super_native_extensions: 85efee3a7495b46b04befcfc86ed12069264ebf3
url_launcher_macos: c82c93949963e55b228a30115bd219499a6fe404
webview_flutter_wkwebview: 0982481e3d9c78fd5c6f62a002fcd24fc791f1e4
window_manager: 990c8e348c4da2a93b81da638245d40554ec9436

PODFILE CHECKSUM: 0532f3f001ca3110b8be345d6491fff690e95823

Expand Down
Loading
Loading