Skip to content

Commit 612d652

Browse files
appflowyappflowy
authored
feat: support local ai embedding, local ai search, local document content search (#7839)
* chore: generate embeddings * chore: save embedding * chore: vec sqlite * chore: clippy * chore: init vector * chore: create vector db * chore: periodically write embedding * chore: fix compile * chore: skip write * chore: impl search * fix: test * fix: stop scheduler * fix: search * chore: add test * chore: update schema * chore: index all * chore: index * chore: search document content * chore: index document content and title * chore: index all view * chore: rename trait * refactor: remove folder index manager * chore: index folder changes * chore: delete folder search * chore: update logs * chore: update logs * chore: search * chore: add search test * chore: fmt * chore: fix test * chore: fix ios build
1 parent 7750f22 commit 612d652

File tree

139 files changed

+4998
-4744
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

139 files changed

+4998
-4744
lines changed

‎.github/workflows/android_ci.yaml.bak‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919
env:
2020
CARGO_TERM_COLOR: always
2121
FLUTTER_VERSION: "3.27.4"
22-
RUST_TOOLCHAIN: "1.81.0"
22+
RUST_TOOLCHAIN: "1.84.0"
2323
CARGO_MAKE_VERSION: "0.37.18"
2424
CLOUD_VERSION: 0.6.54-amd64
2525

‎.github/workflows/flutter_ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ on:
2626
env:
2727
CARGO_TERM_COLOR: always
2828
FLUTTER_VERSION: "3.27.4"
29-
RUST_TOOLCHAIN: "1.81.0"
29+
RUST_TOOLCHAIN: "1.84.0"
3030
CARGO_MAKE_VERSION: "0.37.18"
3131
CLOUD_VERSION: 0.9.37-amd64
3232

‎.github/workflows/ios_ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919

2020
env:
2121
FLUTTER_VERSION: "3.27.4"
22-
RUST_TOOLCHAIN: "1.81.0"
22+
RUST_TOOLCHAIN: "1.84.0"
2323

2424
concurrency:
2525
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

‎.github/workflows/release.yml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ on:
77

88
env:
99
FLUTTER_VERSION: "3.27.4"
10-
RUST_TOOLCHAIN: "1.81.0"
10+
RUST_TOOLCHAIN: "1.84.0"
1111

1212
jobs:
1313
create-release:
@@ -232,10 +232,10 @@ jobs:
232232
matrix:
233233
job:
234234
- {
235-
targets: "aarch64-apple-darwin,x86_64-apple-darwin",
236-
os: macos-latest,
237-
extra-build-args: "",
238-
}
235+
targets: "aarch64-apple-darwin,x86_64-apple-darwin",
236+
os: macos-latest,
237+
extra-build-args: "",
238+
}
239239
steps:
240240
- name: Checkout source code
241241
uses: actions/checkout@v4
@@ -336,12 +336,12 @@ jobs:
336336
matrix:
337337
job:
338338
- {
339-
arch: x86_64,
340-
target: x86_64-unknown-linux-gnu,
341-
os: ubuntu-22.04,
342-
extra-build-args: "",
343-
flutter_profile: production-linux-x86_64,
344-
}
339+
arch: x86_64,
340+
target: x86_64-unknown-linux-gnu,
341+
os: ubuntu-22.04,
342+
extra-build-args: "",
343+
flutter_profile: production-linux-x86_64,
344+
}
345345
steps:
346346
- name: Checkout source code
347347
uses: actions/checkout@v4

‎.github/workflows/rust_ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919
env:
2020
CARGO_TERM_COLOR: always
2121
CLOUD_VERSION: 0.8.3-amd64
22-
RUST_TOOLCHAIN: "1.81.0"
22+
RUST_TOOLCHAIN: "1.84.0"
2323

2424
jobs:
2525
ubuntu-job:

‎.github/workflows/rust_coverage.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ on:
1111
env:
1212
CARGO_TERM_COLOR: always
1313
FLUTTER_VERSION: "3.27.4"
14-
RUST_TOOLCHAIN: "1.81.0"
14+
RUST_TOOLCHAIN: "1.84.0"
1515

1616
jobs:
1717
tests:

‎frontend/Makefile.toml‎

Lines changed: 35 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
#https://github.com/sagiegurari/cargo-make
22

33
extend = [
4-
{ path = "scripts/makefile/desktop.toml" },
5-
{ path = "scripts/makefile/mobile.toml" },
6-
{ path = "scripts/makefile/protobuf.toml" },
7-
{ path = "scripts/makefile/tests.toml" },
8-
{ path = "scripts/makefile/docker.toml" },
9-
{ path = "scripts/makefile/env.toml" },
10-
{ path = "scripts/makefile/flutter.toml" },
11-
{ path = "scripts/makefile/tool.toml" },
12-
{ path = "scripts/makefile/tauri.toml" },
13-
{ path = "scripts/makefile/web.toml" },
4+
{ path = "scripts/makefile/desktop.toml" },
5+
{ path = "scripts/makefile/mobile.toml" },
6+
{ path = "scripts/makefile/protobuf.toml" },
7+
{ path = "scripts/makefile/tests.toml" },
8+
{ path = "scripts/makefile/docker.toml" },
9+
{ path = "scripts/makefile/env.toml" },
10+
{ path = "scripts/makefile/flutter.toml" },
11+
{ path = "scripts/makefile/tool.toml" },
12+
{ path = "scripts/makefile/tauri.toml" },
13+
{ path = "scripts/makefile/web.toml" },
1414
]
1515

1616
[config]
@@ -228,30 +228,31 @@ script = ['''
228228
echo APP_ENVIRONMENT: ${APP_ENVIRONMENT}
229229
echo BUILD_ARCHS: ${BUILD_ARCHS}
230230
echo BUILD_VERSION: ${BUILD_VERSION}
231+
echo RUST_VERSION: "$(rustc --version)"
231232
''']
232233
script_runner = "@shell"
233234

234235
[tasks.setup-crate-type]
235236
private = true
236237
script = [
237-
"""
238-
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
239-
val = replace ${toml} "staticlib" ${CRATE_TYPE}
240-
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
241-
assert ${result}
242-
""",
238+
"""
239+
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
240+
val = replace ${toml} "staticlib" ${CRATE_TYPE}
241+
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
242+
assert ${result}
243+
""",
243244
]
244245
script_runner = "@duckscript"
245246

246247
[tasks.restore-crate-type]
247248
private = true
248249
script = [
249-
"""
250-
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
251-
val = replace ${toml} ${CRATE_TYPE} "staticlib"
252-
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
253-
assert ${result}
254-
""",
250+
"""
251+
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
252+
val = replace ${toml} ${CRATE_TYPE} "staticlib"
253+
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
254+
assert ${result}
255+
""",
255256
]
256257
script_runner = "@duckscript"
257258

@@ -279,24 +280,24 @@ TEST_COMPILE_TARGET = "x86_64-pc-windows-msvc"
279280
[tasks.setup-test-crate-type]
280281
private = true
281282
script = [
282-
"""
283-
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
284-
val = replace ${toml} "staticlib" ${TEST_CRATE_TYPE}
285-
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
286-
assert ${result}
287-
""",
283+
"""
284+
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
285+
val = replace ${toml} "staticlib" ${TEST_CRATE_TYPE}
286+
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
287+
assert ${result}
288+
""",
288289
]
289290
script_runner = "@duckscript"
290291

291292
[tasks.restore-test-crate-type]
292293
private = true
293294
script = [
294-
"""
295-
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
296-
val = replace ${toml} ${TEST_CRATE_TYPE} "staticlib"
297-
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
298-
assert ${result}
299-
""",
295+
"""
296+
toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
297+
val = replace ${toml} ${TEST_CRATE_TYPE} "staticlib"
298+
result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
299+
assert ${result}
300+
""",
300301
]
301302
script_runner = "@duckscript"
302303

‎frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ class CommandPaletteBloc
4949
final TrashService _trashService = TrashService();
5050
final TrashListener _trashListener = TrashListener();
5151
String? _activeQuery;
52-
String? _workspaceId;
5352

5453
@override
5554
Future<void> close() {
@@ -115,7 +114,6 @@ class CommandPaletteBloc
115114
unawaited(
116115
SearchBackendService.performSearch(
117116
event.search,
118-
workspaceId: _workspaceId,
119117
).then(
120118
(result) => result.fold(
121119
(stream) {
@@ -161,6 +159,7 @@ class CommandPaletteBloc
161159
onServerItems: (items, searchId, searching, generatingAIOverview) =>
162160
_handleResultsUpdate(
163161
searchId: searchId,
162+
summaries: [], // when got server search result, summaries should be empty
164163
serverItems: items,
165164
searching: searching,
166165
generatingAIOverview: generatingAIOverview,
@@ -263,7 +262,6 @@ class CommandPaletteBloc
263262
_WorkspaceChanged event,
264263
Emitter<CommandPaletteState> emit,
265264
) {
266-
_workspaceId = event.workspaceId;
267265
emit(
268266
state.copyWith(
269267
query: '',

‎frontend/appflowy_flutter/lib/workspace/application/command_palette/search_service.dart‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,28 +8,23 @@ import 'package:appflowy_backend/protobuf/flowy-error/errors.pb.dart';
88
import 'package:appflowy_backend/protobuf/flowy-search/notification.pb.dart';
99
import 'package:appflowy_backend/protobuf/flowy-search/query.pb.dart';
1010
import 'package:appflowy_backend/protobuf/flowy-search/result.pb.dart';
11-
import 'package:appflowy_backend/protobuf/flowy-search/search_filter.pb.dart';
1211
import 'package:appflowy_result/appflowy_result.dart';
13-
import 'package:nanoid/nanoid.dart';
1412
import 'package:fixnum/fixnum.dart';
1513

1614
class SearchBackendService {
1715
static Future<FlowyResult<SearchResponseStream, FlowyError>> performSearch(
18-
String keyword, {
19-
String? workspaceId,
20-
}) async {
21-
final searchId = nanoid(6);
16+
String keyword,
17+
) async {
18+
final searchId = DateTime.now().millisecondsSinceEpoch.toString();
2219
final stream = SearchResponseStream(searchId: searchId);
2320

24-
final filter = SearchFilterPB(workspaceId: workspaceId);
2521
final request = SearchQueryPB(
2622
search: keyword,
27-
filter: filter,
2823
searchId: searchId,
2924
streamPort: Int64(stream.nativePort),
3025
);
3126

32-
unawaited(SearchEventSearch(request).send());
27+
unawaited(SearchEventStreamSearch(request).send());
3328
return FlowyResult.success(stream);
3429
}
3530
}

‎frontend/appflowy_flutter/macos/Podfile.lock‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -144,34 +144,34 @@ EXTERNAL SOURCES:
144144
:path: Flutter/ephemeral/.symlinks/plugins/window_manager/macos
145145

146146
SPEC CHECKSUMS:
147-
app_links: 9028728e32c83a0831d9db8cf91c526d16cc5468
148-
appflowy_backend: 464aeb3e5c6966a41641a2111e5ead72ce2695f7
149-
auto_updater_macos: 3a42f1a06be6981f1a18be37e6e7bf86aa732118
150-
bitsdojo_window_macos: 7959fb0ca65a3ccda30095c181ecb856fae48ea9
151-
connectivity_plus: e74b9f74717d2d99d45751750e266e55912baeb5
152-
desktop_drop: e0b672a7d84c0a6cbc378595e82cdb15f2970a43
153-
device_info_plus: 4fb280989f669696856f8b129e4a5e3cd6c48f76
154-
file_selector_macos: 6280b52b459ae6c590af5d78fc35c7267a3c4b31
155-
flowy_infra_ui: 8760ff42a789de40bf5007a5f176b454722a341e
147+
app_links: 10e0a0ab602ffaf34d142cd4862f29d34b303b2a
148+
appflowy_backend: 865496343de667fc8c600e04b9fd05234e130cf9
149+
auto_updater_macos: 3e3462c418fe4e731917eacd8d28eef7af84086d
150+
bitsdojo_window_macos: 44e3b8fe3dd463820e0321f6256c5b1c16bb6a00
151+
connectivity_plus: 18d3c32514c886e046de60e9c13895109866c747
152+
desktop_drop: 69eeff437544aa619c8db7f4481b3a65f7696898
153+
device_info_plus: 1b14eed9bf95428983aed283a8d51cce3d8c4215
154+
file_selector_macos: cc3858c981fe6889f364731200d6232dac1d812d
155+
flowy_infra_ui: 03301a39ad118771adbf051a664265c61c507f38
156156
FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24
157157
HotKey: 400beb7caa29054ea8d864c96f5ba7e5b4852277
158-
hotkey_manager: b443f35f4d772162937aa73fd8995e579f8ac4e2
159-
irondash_engine_context: 893c7d96d20ce361d7e996f39d360c4c2f9869ba
160-
local_notifier: ebf072651e35ae5e47280ad52e2707375cb2ae4e
161-
package_info_plus: f0052d280d17aa382b932f399edf32507174e870
162-
path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564
158+
hotkey_manager: c32bf0bfe8f934b7bc17ab4ad5c4c142960b023c
159+
irondash_engine_context: da62996ee25616d2f01bbeb85dc115d813359478
160+
local_notifier: e9506bc66fc70311e8bc7291fb70f743c081e4ff
161+
package_info_plus: 12f1c5c2cfe8727ca46cbd0b26677728972d9a5b
162+
path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46
163163
ReachabilitySwift: 32793e867593cfc1177f5d16491e3a197d2fccda
164-
screen_retriever_macos: 452e51764a9e1cdb74b3c541238795849f21557f
164+
screen_retriever_macos: 776e0fa5d42c6163d2bf772d22478df4b302b161
165165
Sentry: da60d980b197a46db0b35ea12cb8f39af48d8854
166-
sentry_flutter: 27892878729f42701297c628eb90e7c6529f3684
167-
share_plus: 510bf0af1a42cd602274b4629920c9649c52f4cc
168-
shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7
166+
sentry_flutter: 2df8b0aab7e4aba81261c230cbea31c82a62dd1b
167+
share_plus: 1fa619de8392a4398bfaf176d441853922614e89
168+
shared_preferences_foundation: fcdcbc04712aee1108ac7fda236f363274528f78
169169
Sparkle: 9c328bdcfbcaf8f030c4b678eadfd0fcb03822d8
170-
sqflite_darwin: 20b2a3a3b70e43edae938624ce550a3cbf66a3d0
171-
super_native_extensions: c2795d6d9aedf4a79fae25cb6160b71b50549189
172-
url_launcher_macos: 0fba8ddabfc33ce0a9afe7c5fef5aab3d8d2d673
173-
webview_flutter_wkwebview: 44d4dee7d7056d5ad185d25b38404436d56c547c
174-
window_manager: e8d0b1431ab6c454f2b5c9ae26004bbfa43469aa
170+
sqflite_darwin: 5a7236e3b501866c1c9befc6771dfd73ffb8702d
171+
super_native_extensions: 85efee3a7495b46b04befcfc86ed12069264ebf3
172+
url_launcher_macos: c82c93949963e55b228a30115bd219499a6fe404
173+
webview_flutter_wkwebview: 0982481e3d9c78fd5c6f62a002fcd24fc791f1e4
174+
window_manager: 990c8e348c4da2a93b81da638245d40554ec9436
175175

176176
PODFILE CHECKSUM: 0532f3f001ca3110b8be345d6491fff690e95823
177177

0 commit comments

Comments
 (0)