Skip to content

Commit 9b4c200

Browse files
cherylEnkiducherylEnkidu
authored
Add IndexBackfiller to Support Client Side Indexing (#10021)
* Add IndexBackfiller * Address feedback * Address more feedback
1 parent bc2bc6b commit 9b4c200

32 files changed

+1046
-153
lines changed

‎Firestore/Example/Firestore.xcodeproj/project.pbxproj

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@
129129
15BF63DFF3A7E9A5376C4233 /* transform_operation_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 33607A3AE91548BD219EC9C6 /* transform_operation_test.cc */; };
130130
15F54E9538839D56A40C5565 /* watch_change_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2D7472BC70C024D736FF74D9 /* watch_change_test.cc */; };
131131
160B8B6F32963E94CB70B14F /* leveldb_query_engine_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = DB1F1E1B1ED15E8D042144B1 /* leveldb_query_engine_test.cc */; };
132+
167659CDCA47B450F2441454 /* index_backfiller_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */; };
132133
16791B16601204220623916C /* status_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54A0352C20A3B3D7003E0143 /* status_test.cc */; };
133134
16FE432587C1B40AF08613D2 /* objc_type_traits_apple_test.mm in Sources */ = {isa = PBXBuildFile; fileRef = 2A0CF41BA5AED6049B0BEB2C /* objc_type_traits_apple_test.mm */; };
134135
16FF9073CA381CA43CA9BF29 /* FIRTransactionOptionsTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = CF39ECA1293D21A0A2AB2626 /* FIRTransactionOptionsTests.mm */; };
@@ -726,6 +727,7 @@
726727
75C6CECF607CA94F56260BAB /* memory_document_overlay_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 29D9C76922DAC6F710BC1EF4 /* memory_document_overlay_cache_test.cc */; };
727728
75D124966E727829A5F99249 /* FIRTypeTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E071202154D600B64F25 /* FIRTypeTests.mm */; };
728729
76A5447D76F060E996555109 /* task_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 899FC22684B0F7BEEAE13527 /* task_test.cc */; };
730+
76FEBDD2793B729BAD2E84C7 /* index_backfiller_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */; };
729731
7731E564468645A4A62E2A3C /* leveldb_key_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54995F6E205B6E12004EFFA0 /* leveldb_key_test.cc */; };
730732
77BB66DD17A8E6545DE22E0B /* remote_document_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 7EB299CF85034F09CFD6F3FD /* remote_document_cache_test.cc */; };
731733
77C459976DCF7503AEE18F7F /* leveldb_bundle_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 8E9CD82E60893DDD7757B798 /* leveldb_bundle_cache_test.cc */; };
@@ -849,6 +851,7 @@
849851
915A9B8DB280DB4787D83FFE /* byte_stream_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 432C71959255C5DBDF522F52 /* byte_stream_test.cc */; };
850852
91AEFFEE35FBE15FEC42A1F4 /* memory_local_store_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = F6CA0C5638AB6627CB5B4CF4 /* memory_local_store_test.cc */; };
851853
920B6ABF76FDB3547F1CCD84 /* firestore.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 544129D421C2DDC800EFB9CC /* firestore.pb.cc */; };
854+
9236478E01DF2EC7DF58B1FC /* index_backfiller_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */; };
852855
925BE64990449E93242A00A2 /* memory_mutation_queue_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 74FBEFA4FE4B12C435011763 /* memory_mutation_queue_test.cc */; };
853856
92D7081085679497DC112EDB /* persistence_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = 9113B6F513D0473AEABBAF1F /* persistence_testing.cc */; };
854857
92EFF0CC2993B43CBC7A61FF /* grpc_streaming_reader_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6D964922154AB8F00EB9CFB /* grpc_streaming_reader_test.cc */; };
@@ -1037,6 +1040,7 @@
10371040
B83A1416C3922E2F3EBA77FE /* grpc_stream_tester.cc in Sources */ = {isa = PBXBuildFile; fileRef = 87553338E42B8ECA05BA987E /* grpc_stream_tester.cc */; };
10381041
B842780CF42361ACBBB381A9 /* autoid_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54740A521FC913E500713A1A /* autoid_test.cc */; };
10391042
B844B264311E18051B1671ED /* value_util_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 40F9D09063A07F710811A84F /* value_util_test.cc */; };
1043+
B845B9EDED330D0FDAD891BC /* index_backfiller_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */; };
10401044
B896E5DE1CC27347FAC009C3 /* BasicCompileTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DE0761F61F2FE68D003233AF /* BasicCompileTests.swift */; };
10411045
B921A4F35B58925D958DD9A6 /* reference_set_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 132E32997D781B896672D30A /* reference_set_test.cc */; };
10421046
B9706A5CD29195A613CF4147 /* bundle_reader_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 6ECAF7DE28A19C69DF386D88 /* bundle_reader_test.cc */; };
@@ -1119,6 +1123,7 @@
11191123
CBC891BEEC525F4D8F40A319 /* latlng.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE9220B89AAC00B5BCE7 /* latlng.pb.cc */; };
11201124
CBDCA7829AAFEB4853C15517 /* bundle_serializer_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B5C2A94EE24E60543F62CC35 /* bundle_serializer_test.cc */; };
11211125
CC94A33318F983907E9ED509 /* resume_token_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 54DA12A41F315EE100DD57A1 /* resume_token_spec_test.json */; };
1126+
CCE596E8654A4D2EEA75C219 /* index_backfiller_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */; };
11221127
CD0AA9E5D83C00CAAE7C2F67 /* FIRTimestampTest.m in Sources */ = {isa = PBXBuildFile; fileRef = B65D34A7203C99090076A5E1 /* FIRTimestampTest.m */; };
11231128
CD1E2F356FC71D7E74FCD26C /* leveldb_remote_document_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 0840319686A223CC4AD3FAB1 /* leveldb_remote_document_cache_test.cc */; };
11241129
CD226D868CEFA9D557EF33A1 /* query_listener_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 7C3F995E040E9E9C5E8514BB /* query_listener_test.cc */; };
@@ -1215,6 +1220,7 @@
12151220
E1264B172412967A09993EC6 /* byte_string_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 5342CDDB137B4E93E2E85CCA /* byte_string_test.cc */; };
12161221
E186D002520881AD2906ADDB /* status.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE9920B89AAC00B5BCE7 /* status.pb.cc */; };
12171222
E21D819A06D9691A4B313440 /* remote_store_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 3B843E4A1F3930A400548890 /* remote_store_spec_test.json */; };
1223+
E25DCFEF318E003B8B7B9DC8 /* index_backfiller_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */; };
12181224
E27C0996AF6EC6D08D91B253 /* document.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 544129D821C2DDC800EFB9CC /* document.pb.cc */; };
12191225
E2AE851F9DC4C037CCD05E36 /* remote_document_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 7EB299CF85034F09CFD6F3FD /* remote_document_cache_test.cc */; };
12201226
E2B15548A3B6796CE5A01975 /* FIRListenerRegistrationTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E06B202154D500B64F25 /* FIRListenerRegistrationTests.mm */; };
@@ -1421,6 +1427,7 @@
14211427
1C01D8CE367C56BB2624E299 /* index.pb.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; name = index.pb.h; path = admin/index.pb.h; sourceTree = "<group>"; };
14221428
1C3F7302BF4AE6CBC00ECDD0 /* resource.pb.cc */ = {isa = PBXFileReference; includeInIndex = 1; path = resource.pb.cc; sourceTree = "<group>"; };
14231429
1CA9800A53669EFBFFB824E3 /* memory_remote_document_cache_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = memory_remote_document_cache_test.cc; sourceTree = "<group>"; };
1430+
1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; path = index_backfiller_test.cc; sourceTree = "<group>"; };
14241431
214877F52A705012D6720CA0 /* object_value_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = object_value_test.cc; sourceTree = "<group>"; };
14251432
2220F583583EFC28DE792ABE /* Pods_Firestore_IntegrationTests_tvOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Firestore_IntegrationTests_tvOS.framework; sourceTree = BUILT_PRODUCTS_DIR; };
14261433
2286F308EFB0534B1BDE05B9 /* memory_target_cache_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = memory_target_cache_test.cc; sourceTree = "<group>"; };
@@ -2098,6 +2105,7 @@
20982105
75E24C5CD7BC423D48713100 /* counting_query_engine.h */,
20992106
FFCA39825D9678A03D1845D0 /* document_overlay_cache_test.cc */,
21002107
DF445D5201750281F1817387 /* document_overlay_cache_test.h */,
2108+
1F50E872B3F117A674DA8E94 /* index_backfiller_test.cc */,
21012109
AE4A9E38D65688EE000EE2A1 /* index_manager_test.cc */,
21022110
73F1F73A2210F3D800E1F692 /* index_manager_test.h */,
21032111
8E9CD82E60893DDD7757B798 /* leveldb_bundle_cache_test.cc */,
@@ -3642,6 +3650,7 @@
36423650
E82F8EBBC8CC37299A459E73 /* hashing_test_apple.mm in Sources */,
36433651
897F3C1936612ACB018CA1DD /* http.pb.cc in Sources */,
36443652
48BC5801432127A90CFF55E3 /* index.pb.cc in Sources */,
3653+
167659CDCA47B450F2441454 /* index_backfiller_test.cc in Sources */,
36453654
FAD97B82766AEC29B7B5A1B7 /* index_manager_test.cc in Sources */,
36463655
E084921EFB7CF8CB1E950D6C /* iterator_adaptors_test.cc in Sources */,
36473656
49C04B97AB282FFA82FD98CD /* latlng.pb.cc in Sources */,
@@ -3846,6 +3855,7 @@
38463855
CD78EEAA1CD36BE691CA3427 /* hashing_test_apple.mm in Sources */,
38473856
1357806B4CD3A62A8F5DE86D /* http.pb.cc in Sources */,
38483857
190F9885BAA81587F08CD26C /* index.pb.cc in Sources */,
3858+
B845B9EDED330D0FDAD891BC /* index_backfiller_test.cc in Sources */,
38493859
F58A23FEF328EB74F681FE83 /* index_manager_test.cc in Sources */,
38503860
0E4C94369FFF7EC0C9229752 /* iterator_adaptors_test.cc in Sources */,
38513861
0FBDD5991E8F6CD5F8542474 /* latlng.pb.cc in Sources */,
@@ -4064,6 +4074,7 @@
40644074
3B256CCF6AEEE12E22F16BB8 /* hashing_test_apple.mm in Sources */,
40654075
AB8209455BAA17850D5E196D /* http.pb.cc in Sources */,
40664076
096BA3A3703AC1491F281618 /* index.pb.cc in Sources */,
4077+
9236478E01DF2EC7DF58B1FC /* index_backfiller_test.cc in Sources */,
40674078
4BFEEB7FDD7CD5A693B5B5C1 /* index_manager_test.cc in Sources */,
40684079
FA334ADC73CFDB703A7C17CD /* iterator_adaptors_test.cc in Sources */,
40694080
CBC891BEEC525F4D8F40A319 /* latlng.pb.cc in Sources */,
@@ -4282,6 +4293,7 @@
42824293
BDD2D1812BAD962E3C81A53F /* hashing_test_apple.mm in Sources */,
42834294
49794806F3D5052E5F61A40D /* http.pb.cc in Sources */,
42844295
6E8CD8F545C8EDA84918977C /* index.pb.cc in Sources */,
4296+
E25DCFEF318E003B8B7B9DC8 /* index_backfiller_test.cc in Sources */,
42854297
650B31A5EC6F8D2AEA79C350 /* index_manager_test.cc in Sources */,
42864298
86494278BE08F10A8AAF9603 /* iterator_adaptors_test.cc in Sources */,
42874299
4173B61CB74EB4CD1D89EE68 /* latlng.pb.cc in Sources */,
@@ -4496,6 +4508,7 @@
44964508
B69CF3F12227386500B281C8 /* hashing_test_apple.mm in Sources */,
44974509
618BBEB020B89AAC00B5BCE7 /* http.pb.cc in Sources */,
44984510
77D38E78F7CCB8504450A8FB /* index.pb.cc in Sources */,
4511+
76FEBDD2793B729BAD2E84C7 /* index_backfiller_test.cc in Sources */,
44994512
E6357221227031DD77EE5265 /* index_manager_test.cc in Sources */,
45004513
54A0353520A3D8CB003E0143 /* iterator_adaptors_test.cc in Sources */,
45014514
618BBEAE20B89AAC00B5BCE7 /* latlng.pb.cc in Sources */,
@@ -4733,6 +4746,7 @@
47334746
433474A3416B76645FFD17BB /* hashing_test_apple.mm in Sources */,
47344747
06A3926F89C847846BE4D6BE /* http.pb.cc in Sources */,
47354748
78E8DDDBE131F3DA9AF9F8B8 /* index.pb.cc in Sources */,
4749+
CCE596E8654A4D2EEA75C219 /* index_backfiller_test.cc in Sources */,
47364750
2B4234B962625F9EE68B31AC /* index_manager_test.cc in Sources */,
47374751
8A79DDB4379A063C30A76329 /* iterator_adaptors_test.cc in Sources */,
47384752
23C04A637090E438461E4E70 /* latlng.pb.cc in Sources */,

‎Firestore/core/src/core/firestore_client.cc

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,20 @@ using util::StatusOrCallback;
101101
using util::ThrowIllegalState;
102102
using util::TimerId;
103103

104+
namespace {
105+
104106
static const size_t kMaxConcurrentLimboResolutions = 100;
105107

108+
static const auto kInitialGCDelay = std::chrono::minutes(1);
109+
static const auto kRegularGCDelay = std::chrono::minutes(5);
110+
111+
/** How long we wait to try running index backfill after SDK initialization. */
112+
static const auto kInitialBackfillDelay = std::chrono::milliseconds(15);
113+
/** Minimum amount of time between backfill checks, after the first one. */
114+
static const auto kRegularBackfillDelay = std::chrono::milliseconds(1);
115+
116+
} // namespace
117+
106118
std::shared_ptr<FirestoreClient> FirestoreClient::Create(
107119
const DatabaseInfo& database_info,
108120
const api::Settings& settings,
@@ -236,6 +248,8 @@ void FirestoreClient::Initialize(const User& user, const Settings& settings) {
236248
// refilling mutation queue, etc.) so must be started after LocalStore.
237249
local_store_->Start();
238250
remote_store_->Start();
251+
252+
ScheduleIndexBackfiller();
239253
}
240254

241255
FirestoreClient::~FirestoreClient() {
@@ -296,6 +310,8 @@ void FirestoreClient::TerminateInternal() {
296310
// If we've scheduled LRU garbage collection, cancel it.
297311
lru_callback_.Cancel();
298312

313+
backfiller_callback_.Cancel();
314+
299315
remote_store_->Shutdown();
300316
persistence_->Shutdown();
301317

@@ -307,13 +323,9 @@ void FirestoreClient::TerminateInternal() {
307323
remote_store_.reset();
308324
}
309325

310-
/**
311-
* Schedules a callback to try running LRU garbage collection. Reschedules
312-
* itself after the GC has run.
313-
*/
314326
void FirestoreClient::ScheduleLruGarbageCollection() {
315327
std::chrono::milliseconds delay =
316-
gc_has_run_ ? regular_gc_delay_ : initial_gc_delay_;
328+
gc_has_run_ ? kRegularGCDelay : kInitialGCDelay;
317329

318330
lru_callback_ = worker_queue_->EnqueueAfterDelay(
319331
delay, TimerId::GarbageCollectionDelay, [this] {
@@ -323,6 +335,18 @@ void FirestoreClient::ScheduleLruGarbageCollection() {
323335
});
324336
}
325337

338+
void FirestoreClient::ScheduleIndexBackfiller() {
339+
std::chrono::milliseconds delay =
340+
backfiller_has_run_ ? kRegularBackfillDelay : kInitialBackfillDelay;
341+
342+
backfiller_callback_ = worker_queue_->EnqueueAfterDelay(
343+
delay, TimerId::IndexBackfillDelay, [this] {
344+
local_store_->Backfill();
345+
backfiller_has_run_ = true;
346+
ScheduleIndexBackfiller();
347+
});
348+
}
349+
326350
void FirestoreClient::DisableNetwork(StatusCallback callback) {
327351
VerifyNotTerminated();
328352

‎Firestore/core/src/core/firestore_client.h

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,18 @@ class FirestoreClient : public std::enable_shared_from_this<FirestoreClient> {
204204

205205
void TerminateInternal();
206206

207+
/**
208+
* Schedules a callback to try running LRU garbage collection. Reschedules
209+
* itself after the GC has run.
210+
*/
207211
void ScheduleLruGarbageCollection();
208212

213+
/**
214+
* Schedules a callback to try running index backfiller. Reschedules
215+
* itself after the backfiller has run.
216+
*/
217+
void ScheduleIndexBackfiller();
218+
209219
DatabaseInfo database_info_;
210220
std::shared_ptr<credentials::AppCheckCredentialsProvider>
211221
app_check_credentials_provider_;
@@ -231,12 +241,12 @@ class FirestoreClient : public std::enable_shared_from_this<FirestoreClient> {
231241
std::unique_ptr<SyncEngine> sync_engine_;
232242
std::unique_ptr<EventManager> event_manager_;
233243

234-
std::chrono::milliseconds initial_gc_delay_ = std::chrono::minutes(1);
235-
std::chrono::milliseconds regular_gc_delay_ = std::chrono::minutes(5);
236244
bool gc_has_run_ = false;
245+
bool backfiller_has_run_ = false;
237246
bool credentials_initialized_ = false;
238247
local::LruDelegate* _Nullable lru_delegate_;
239248
util::DelayedOperation lru_callback_;
249+
util::DelayedOperation backfiller_callback_;
240250
};
241251

242252
} // namespace core

‎Firestore/core/src/local/document_overlay_cache.cc

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
#include "Firestore/core/src/local/document_overlay_cache.h"
1818

19+
#include <set>
1920
#include <utility>
2021

2122
#include "Firestore/core/src/immutable/sorted_set.h"
@@ -31,8 +32,8 @@ using model::DocumentKeySet;
3132
using model::Overlay;
3233
using model::OverlayByDocumentKeyMap;
3334

34-
void DocumentOverlayCache::GetOverlays(OverlayByDocumentKeyMap& dest,
35-
const DocumentKeySet& keys) const {
35+
void DocumentOverlayCache::GetOverlays(
36+
OverlayByDocumentKeyMap& dest, const std::set<DocumentKey>& keys) const {
3637
for (const DocumentKey& key : keys) {
3738
absl::optional<Overlay> overlay = GetOverlay(key);
3839
if (overlay.has_value()) {

‎Firestore/core/src/local/document_overlay_cache.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#define FIRESTORE_CORE_SRC_LOCAL_DOCUMENT_OVERLAY_CACHE_H_
1919

2020
#include <cstdlib>
21+
#include <set>
2122
#include <unordered_map>
2223

2324
#include "Firestore/core/src/model/document_key.h"
@@ -60,7 +61,7 @@ class DocumentOverlayCache {
6061
* which there are no overlays.
6162
*/
6263
virtual void GetOverlays(model::OverlayByDocumentKeyMap& dest,
63-
const model::DocumentKeySet& keys) const;
64+
const std::set<model::DocumentKey>& keys) const;
6465

6566
/**
6667
* Saves the given document key to mutation map to persistence as overlays.

0 commit comments

Comments
 (0)