Skip to content

Commit 1a023a3

Browse files
dconeybedconeybe
authored
Improve performance of LevelDbDocumentOverlayCache::GetOverlays(ResourcePath) (#9401)
1 parent 517ce6c commit 1a023a3

File tree

6 files changed

+434
-121
lines changed

6 files changed

+434
-121
lines changed

‎Firestore/core/src/local/leveldb_document_overlay_cache.cc

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ absl::optional<Overlay> LevelDbDocumentOverlayCache::GetOverlay(
6060
auto it = db_->current_transaction()->NewIterator();
6161
it->Seek(key_prefix);
6262

63-
if (!(it->Valid() && absl::StartsWith(it->key(), key_prefix))) {
63+
if (!it->Valid() || !absl::StartsWith(it->key(), key_prefix)) {
6464
return absl::nullopt;
6565
}
6666

@@ -88,28 +88,13 @@ void LevelDbDocumentOverlayCache::RemoveOverlaysForBatchId(int batch_id) {
8888
DocumentOverlayCache::OverlayByDocumentKeyMap
8989
LevelDbDocumentOverlayCache::GetOverlays(const ResourcePath& collection,
9090
int since_batch_id) const {
91-
// TODO(dconeybe) Implement an index so that this query can be performed
92-
// without requiring a full table scan.
93-
9491
OverlayByDocumentKeyMap result;
95-
96-
const size_t immediate_children_path_length{collection.size() + 1};
97-
98-
ForEachOverlay([&](LevelDbDocumentOverlayKey&& key,
99-
absl::string_view encoded_mutation) {
100-
if (!collection.IsPrefixOf(key.document_key().path())) {
101-
return;
102-
}
103-
// Documents from sub-collections
104-
if (key.document_key().path().size() != immediate_children_path_length) {
105-
return;
106-
}
107-
108-
if (key.largest_batch_id() > since_batch_id) {
109-
result[key.document_key()] = ParseOverlay(key, encoded_mutation);
110-
}
111-
});
112-
92+
ForEachKeyInCollection(
93+
collection, since_batch_id, [&](LevelDbDocumentOverlayKey&& key) {
94+
absl::optional<Overlay> overlay = GetOverlay(key);
95+
HARD_ASSERT(overlay.has_value());
96+
result[std::move(key).document_key()] = std::move(overlay).value();
97+
});
11398
return result;
11499
}
115100

@@ -167,6 +152,11 @@ int LevelDbDocumentOverlayCache::GetLargestBatchIdIndexEntryCount() const {
167152
LevelDbDocumentOverlayLargestBatchIdIndexKey::KeyPrefix(user_id_));
168153
}
169154

155+
int LevelDbDocumentOverlayCache::GetCollectionIndexEntryCount() const {
156+
return CountEntriesWithKeyPrefix(
157+
LevelDbDocumentOverlayCollectionIndexKey::KeyPrefix(user_id_));
158+
}
159+
170160
int LevelDbDocumentOverlayCache::CountEntriesWithKeyPrefix(
171161
const std::string& key_prefix) const {
172162
int count = 0;
@@ -202,6 +192,7 @@ void LevelDbDocumentOverlayCache::SaveOverlay(int largest_batch_id,
202192
auto* transaction = db_->current_transaction();
203193
transaction->Put(key.Encode(), serializer_->EncodeMutation(mutation));
204194
transaction->Put(LevelDbDocumentOverlayLargestBatchIdIndexKey::Key(key), "");
195+
transaction->Put(LevelDbDocumentOverlayCollectionIndexKey::Key(key), "");
205196
}
206197

207198
void LevelDbDocumentOverlayCache::DeleteOverlay(
@@ -211,7 +202,7 @@ void LevelDbDocumentOverlayCache::DeleteOverlay(
211202
auto it = db_->current_transaction()->NewIterator();
212203
it->Seek(key_prefix);
213204

214-
if (!(it->Valid() && absl::StartsWith(it->key(), key_prefix))) {
205+
if (!it->Valid() || !absl::StartsWith(it->key(), key_prefix)) {
215206
return;
216207
}
217208

@@ -227,6 +218,7 @@ void LevelDbDocumentOverlayCache::DeleteOverlay(
227218
auto* transaction = db_->current_transaction();
228219
transaction->Delete(key.Encode());
229220
transaction->Delete(LevelDbDocumentOverlayLargestBatchIdIndexKey::Key(key));
221+
transaction->Delete(LevelDbDocumentOverlayCollectionIndexKey::Key(key));
230222
}
231223

232224
void LevelDbDocumentOverlayCache::ForEachOverlay(
@@ -258,6 +250,40 @@ void LevelDbDocumentOverlayCache::ForEachKeyWithLargestBatchId(
258250
}
259251
}
260252

253+
void LevelDbDocumentOverlayCache::ForEachKeyInCollection(
254+
const ResourcePath& collection,
255+
int since_batch_id,
256+
std::function<void(LevelDbDocumentOverlayKey&&)> callback) const {
257+
const std::string index_start_key =
258+
LevelDbDocumentOverlayCollectionIndexKey::KeyPrefix(user_id_, collection,
259+
since_batch_id + 1);
260+
const std::string index_key_prefix =
261+
LevelDbDocumentOverlayCollectionIndexKey::KeyPrefix(user_id_, collection);
262+
263+
auto it = db_->current_transaction()->NewIterator();
264+
for (it->Seek(index_start_key);
265+
it->Valid() && absl::StartsWith(it->key(), index_key_prefix);
266+
it->Next()) {
267+
LevelDbDocumentOverlayCollectionIndexKey key;
268+
HARD_ASSERT(key.Decode(it->key()));
269+
if (key.collection() != collection) {
270+
break;
271+
}
272+
callback(std::move(key).ToLevelDbDocumentOverlayKey());
273+
}
274+
}
275+
276+
absl::optional<Overlay> LevelDbDocumentOverlayCache::GetOverlay(
277+
const LevelDbDocumentOverlayKey& key) const {
278+
auto it = db_->current_transaction()->NewIterator();
279+
const std::string encoded_key = key.Encode();
280+
it->Seek(encoded_key);
281+
if (!it->Valid() || it->key() != encoded_key) {
282+
return absl::nullopt;
283+
}
284+
return ParseOverlay(key, it->value());
285+
}
286+
261287
} // namespace local
262288
} // namespace firestore
263289
} // namespace firebase

‎Firestore/core/src/local/leveldb_document_overlay_cache.h

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,10 @@ class LevelDbDocumentOverlayCache final : public DocumentOverlayCache {
7070
private:
7171
friend class LevelDbDocumentOverlayCacheTestHelper;
7272

73-
// Returns the number of index entries for the largest batch ID.
74-
// This method exists for unit testing only.
73+
// Returns the number of index entries of the various types.
74+
// These methods exist for unit testing only.
7575
int GetLargestBatchIdIndexEntryCount() const;
76+
int GetCollectionIndexEntryCount() const;
7677

7778
int GetOverlayCount() const override;
7879
int CountEntriesWithKeyPrefix(const std::string& key_prefix) const;
@@ -97,6 +98,14 @@ class LevelDbDocumentOverlayCache final : public DocumentOverlayCache {
9798
int largest_batch_id,
9899
std::function<void(LevelDbDocumentOverlayKey&&)>) const;
99100

101+
void ForEachKeyInCollection(
102+
const model::ResourcePath& collection,
103+
int since_batch_id,
104+
std::function<void(LevelDbDocumentOverlayKey&&)>) const;
105+
106+
absl::optional<model::mutation::Overlay> GetOverlay(
107+
const LevelDbDocumentOverlayKey& decoded_key) const;
108+
100109
// The LevelDbDocumentOverlayCache instance is owned by LevelDbPersistence.
101110
LevelDbPersistence* db_;
102111

‎Firestore/core/src/local/leveldb_key.cc

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ const char* kIndexEntriesTable = "index_entries";
5656
const char* kDocumentOverlaysTable = "document_overlays";
5757
const char* kDocumentOverlaysLargestBatchIdIndexTable =
5858
"document_overlays_largest_batch_id_index";
59+
const char* kDocumentOverlaysCollectionIndexTable =
60+
"document_overlays_collection_index";
5961

6062
/**
6163
* Labels for the components of keys. These serve to make keys self-describing.
@@ -1262,12 +1264,6 @@ bool LevelDbIndexEntryKey::Decode(absl::string_view key) {
12621264
return reader.ok();
12631265
}
12641266

1265-
std::string LevelDbDocumentOverlayKey::KeyPrefix() {
1266-
Writer writer;
1267-
writer.WriteTableName(kDocumentOverlaysTable);
1268-
return writer.result();
1269-
}
1270-
12711267
std::string LevelDbDocumentOverlayKey::KeyPrefix(absl::string_view user_id) {
12721268
Writer writer;
12731269
writer.WriteTableName(kDocumentOverlaysTable);
@@ -1306,12 +1302,6 @@ bool LevelDbDocumentOverlayKey::Decode(absl::string_view key) {
13061302
return reader.ok();
13071303
}
13081304

1309-
std::string LevelDbDocumentOverlayLargestBatchIdIndexKey::KeyPrefix() {
1310-
Writer writer;
1311-
writer.WriteTableName(kDocumentOverlaysLargestBatchIdIndexTable);
1312-
return writer.result();
1313-
}
1314-
13151305
std::string LevelDbDocumentOverlayLargestBatchIdIndexKey::KeyPrefix(
13161306
absl::string_view user_id) {
13171307
Writer writer;
@@ -1346,10 +1336,68 @@ bool LevelDbDocumentOverlayLargestBatchIdIndexKey::Decode(
13461336
absl::string_view key) {
13471337
Reader reader{key};
13481338
reader.ReadTableNameMatching(kDocumentOverlaysLargestBatchIdIndexTable);
1349-
user_id_ = reader.ReadUserId();
1350-
largest_batch_id_ = reader.ReadBatchId();
1351-
document_key_ = reader.ReadDocumentKey();
1339+
auto user_id = reader.ReadUserId();
1340+
auto largest_batch_id = reader.ReadBatchId();
1341+
auto document_key = reader.ReadDocumentKey();
1342+
reader.ReadTerminator();
1343+
Reset(std::move(user_id), largest_batch_id, std::move(document_key));
1344+
return reader.ok();
1345+
}
1346+
1347+
std::string LevelDbDocumentOverlayCollectionIndexKey::KeyPrefix(
1348+
absl::string_view user_id) {
1349+
Writer writer;
1350+
writer.WriteTableName(kDocumentOverlaysCollectionIndexTable);
1351+
writer.WriteUserId(user_id);
1352+
return writer.result();
1353+
}
1354+
1355+
std::string LevelDbDocumentOverlayCollectionIndexKey::KeyPrefix(
1356+
absl::string_view user_id, const model::ResourcePath& collection) {
1357+
Writer writer;
1358+
writer.WriteTableName(kDocumentOverlaysCollectionIndexTable);
1359+
writer.WriteUserId(user_id);
1360+
writer.WriteResourcePath(collection);
1361+
return writer.result();
1362+
}
1363+
1364+
std::string LevelDbDocumentOverlayCollectionIndexKey::KeyPrefix(
1365+
absl::string_view user_id,
1366+
const model::ResourcePath& collection,
1367+
model::BatchId largest_batch_id) {
1368+
Writer writer;
1369+
writer.WriteTableName(kDocumentOverlaysCollectionIndexTable);
1370+
writer.WriteUserId(user_id);
1371+
writer.WriteResourcePath(collection);
1372+
writer.WriteBatchId(largest_batch_id);
1373+
return writer.result();
1374+
}
1375+
1376+
std::string LevelDbDocumentOverlayCollectionIndexKey::Key(
1377+
absl::string_view user_id,
1378+
const model::ResourcePath& collection,
1379+
model::BatchId largest_batch_id,
1380+
absl::string_view document_id) {
1381+
Writer writer;
1382+
writer.WriteTableName(kDocumentOverlaysCollectionIndexTable);
1383+
writer.WriteUserId(user_id);
1384+
writer.WriteResourcePath(collection);
1385+
writer.WriteBatchId(largest_batch_id);
1386+
writer.WriteDocumentId(document_id);
1387+
writer.WriteTerminator();
1388+
return writer.result();
1389+
}
1390+
1391+
bool LevelDbDocumentOverlayCollectionIndexKey::Decode(absl::string_view key) {
1392+
Reader reader{key};
1393+
reader.ReadTableNameMatching(kDocumentOverlaysCollectionIndexTable);
1394+
auto user_id = reader.ReadUserId();
1395+
const ResourcePath collection = reader.ReadResourcePath();
1396+
auto largest_batch_id = reader.ReadBatchId();
1397+
const std::string document_id = reader.ReadDocumentId();
13521398
reader.ReadTerminator();
1399+
Reset(std::move(user_id), largest_batch_id,
1400+
DocumentKey(collection.Append(document_id)));
13531401
return reader.ok();
13541402
}
13551403

0 commit comments

Comments
 (0)