Skip to content

Commit 9fd7bdc

Browse files
jimwwalkerdaverigby
authored andcommitted
MB-54516: Drop ModifyCollection events
If a collection is dropped, the modify event must be purged along with any normal collection items. All of the dropKey paths were setup to ignore any system events, this is no longer true with the new event. Change-Id: Ia89d4b7a56e9f3b5b23dd176e5d00da88bd83f3e Reviewed-on: https://review.couchbase.org/c/kv_engine/+/183265 Well-Formed: Restriction Checker Tested-by: Build Bot <[email protected]> Reviewed-by: Dave Rigby <[email protected]>
1 parent f41c6ca commit 9fd7bdc

File tree

12 files changed

+128
-25
lines changed

12 files changed

+128
-25
lines changed

engines/ep/src/collections/flush_accounting.cc

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,6 @@
1515

1616
namespace Collections::VB {
1717

18-
// Returns two optionals
19-
// First is only initialised if the key is a collection system event.
20-
// Second is only initialised if there is a collection id present.
21-
// Any mutation will return nullopt,cid
22-
// A collection create/drop/modify will return event, cid
23-
// A scope event will return nullopt, nullopt
24-
2518
/**
2619
* From a DocKey extract CollectionID and SystemEvent data.
2720
* The function returns two optionals because depending on the key there could

engines/ep/src/collections/scan_context.cc

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "collections/scan_context.h"
1313

1414
#include "collections/kvstore.h"
15+
#include "systemevent_factory.h"
1516

1617
namespace Collections::VB {
1718

@@ -29,13 +30,23 @@ ScanContext::ScanContext(
2930
}
3031

3132
bool ScanContext::isLogicallyDeleted(const DocKey& key, uint64_t seqno) const {
32-
if (dropped.empty() || key.isInSystemCollection()) {
33+
if (dropped.empty()) {
3334
return false;
3435
}
3536

37+
CollectionID cid;
38+
if (key.isInSystemCollection()) {
39+
auto modify = SystemEventFactory::isModifyCollection(key);
40+
if (!modify) {
41+
return false;
42+
}
43+
cid = modify.value();
44+
} else {
45+
cid = key.getCollectionID();
46+
}
47+
3648
// Is the key in a range which contains dropped collections and in the set?
37-
return (seqno >= startSeqno && seqno <= endSeqno) &&
38-
dropped.count(key.getCollectionID()) > 0;
49+
return (seqno >= startSeqno && seqno <= endSeqno) && dropped.count(cid) > 0;
3950
}
4051

4152
std::ostream& operator<<(std::ostream& os, const ScanContext& scanContext) {

engines/ep/src/ep_bucket.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1250,7 +1250,8 @@ void EPBucket::dropKey(VBucket& vb,
12501250

12511251
auto docKey = diskKey.getDocKey();
12521252
if (docKey.isInSystemCollection()) {
1253-
throw std::logic_error("EPBucket::dropKey called for a system key");
1253+
// SystemEvents aren't in memory so return.
1254+
return;
12541255
}
12551256

12561257
if (diskKey.isPrepared() && bySeqno > highCompletedSeqno) {

engines/ep/src/ephemeral_vb.cc

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -310,10 +310,16 @@ std::optional<SequenceList::RangeIterator> EphemeralVBucket::makeRangeIterator(
310310

311311
bool EphemeralVBucket::isKeyLogicallyDeleted(const DocKey& key,
312312
int64_t bySeqno) const {
313-
if (key.isInSystemCollection()) {
313+
if (key.isInSystemCollection() &&
314+
!SystemEventFactory::isModifyCollection(key)) {
314315
return false;
316+
// else Modify events can be dropped
315317
}
316-
auto cHandle = lockCollections(key);
318+
319+
// ReadLock the manifest and permit system keys - this ensures a Modify
320+
// can lookup the correct collection
321+
auto cHandle =
322+
manifest->lock(key, Collections::VB::Manifest::AllowSystemKeys{});
317323
return !cHandle.valid() || cHandle.isLogicallyDeleted(bySeqno);
318324
}
319325

@@ -929,11 +935,6 @@ size_t EphemeralVBucket::getNumPersistedDeletes() const {
929935
}
930936

931937
void EphemeralVBucket::dropKey(const DocKey& key, int64_t bySeqno) {
932-
// The system event doesn't get dropped here (tombstone purger will deal)
933-
if (key.isInSystemCollection()) {
934-
return;
935-
}
936-
937938
// if there is a pending item which will need removing from the DM,
938939
// store its seqno here for use after the HT lock has been released.
939940
int64_t prepareSeqno = 0;

engines/ep/src/kvstore/couch-kvstore/couch-kvstore.cc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -935,7 +935,9 @@ static int time_purge_hook(Db& d,
935935

936936
// Track nothing for the individual collection as the stats doc has
937937
// already been deleted.
938-
} else {
938+
} else if (!docKey.isInSystemCollection()) {
939+
// item in a collection was dropped, track how many for item count
940+
// adjustment.
939941
if (!info->deleted) {
940942
ctx.stats.collectionsItemsPurged++;
941943
} else {

engines/ep/src/stored-value.cc

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -479,23 +479,30 @@ static std::string getSystemEventsValueFromStoredValue(const StoredValue& sv) {
479479
std::string_view itemValue{sv.getValue()->getData(),
480480
sv.getValue()->getSize()};
481481

482-
if (systemEventType == SystemEvent::Scope) {
482+
switch (systemEventType) {
483+
case SystemEvent::Scope: {
483484
if (sv.isDeleted()) {
484485
auto eventData = Manifest::getDropScopeEventData(itemValue);
485486
return to_string(eventData);
486487
} else {
487488
auto eventData = Manifest::getCreateScopeEventData(itemValue);
488489
return to_string(eventData);
489490
}
490-
} else if (systemEventType == SystemEvent::Collection) {
491+
break;
492+
}
493+
case SystemEvent::Collection:
494+
case SystemEvent::ModifyCollection: {
491495
if (sv.isDeleted()) {
492496
auto eventData = Manifest::getDropEventData(itemValue);
493497
return to_string(eventData);
494498
} else {
495499
auto eventData = Manifest::getCreateEventData(itemValue);
496500
return to_string(eventData);
497501
}
502+
break;
498503
}
504+
}
505+
499506
throw std::invalid_argument(
500507
"getSystemEventsValueFromStoredValue(): StoredValue must be a "
501508
"SystemEvent for a collection or scope");

engines/ep/src/systemevent_factory.cc

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,12 @@ StoredDocKey SystemEventFactory::makeCollectionEventKey(CollectionID cid,
8585

8686
CollectionID SystemEventFactory::getCollectionIDFromKey(const DocKey& key) {
8787
// Input key is made up of a sequence of prefixes as per makeCollectionEvent
88-
// or makeScopeEvent
88+
// or makeModifyCollectionEvent
8989
// This function skips (1), checks (2) and returns 3
9090
auto se = getSystemEventType(key);
91-
Expects(se.first == SystemEvent::Collection); // expected Collection
91+
// expected Collection/ModifyCollection event
92+
Expects(se.first == SystemEvent::Collection ||
93+
se.first == SystemEvent::ModifyCollection);
9294
return cb::mcbp::unsigned_leb128<CollectionIDType>::decode(se.second).first;
9395
}
9496

@@ -124,6 +126,15 @@ std::pair<SystemEvent, uint32_t> SystemEventFactory::getTypeAndID(
124126
return {type, cb::mcbp::unsigned_leb128<uint32_t>::decode(buffer).first};
125127
}
126128

129+
std::optional<CollectionID> SystemEventFactory::isModifyCollection(
130+
const DocKey& key) {
131+
auto [event, id] = SystemEventFactory::getTypeAndID(key);
132+
if (event != SystemEvent::ModifyCollection) {
133+
return {};
134+
}
135+
return CollectionID(id);
136+
}
137+
127138
std::unique_ptr<SystemEventProducerMessage> SystemEventProducerMessage::make(
128139
uint32_t opaque, queued_item& item, cb::mcbp::DcpStreamId sid) {
129140
// Always ensure decompressed as we are about to use the value

engines/ep/src/systemevent_factory.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,13 @@ class SystemEventFactory {
107107
*/
108108
static std::pair<SystemEvent, uint32_t> getTypeAndID(const DocKey& key);
109109

110+
/**
111+
* Given a key which the caller knows isSystemCollection return an optional
112+
* which stores the ID of the modified collection iff the key represent a
113+
* ModifyCollection event.
114+
*/
115+
static std::optional<CollectionID> isModifyCollection(const DocKey& key);
116+
110117
private:
111118
/**
112119
* Make an Item representing the SystemEvent

engines/ep/tests/module_tests/checkpoint_test.cc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2694,6 +2694,9 @@ TEST_P(CheckpointTest, CheckpointItemToString) {
26942694
auto event =
26952695
SystemEventFactory::makeCollectionEvent(CollectionID(99), {}, {});
26962696
EXPECT_EQ("cid:0x1:0x0:0x63:_collection", event->getKey().to_string());
2697+
event = SystemEventFactory::makeModifyCollectionEvent(
2698+
CollectionID(999), {}, {});
2699+
EXPECT_EQ("cid:0x1:0x2:0x3e7:_collection", event->getKey().to_string());
26972700
event = SystemEventFactory::makeScopeEvent(ScopeID(99), {}, {});
26982701
EXPECT_EQ("cid:0x1:0x1:0x63:_scope", event->getKey().to_string());
26992702
}

engines/ep/tests/module_tests/collections/evp_store_collections_eraser_test.cc

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2177,6 +2177,65 @@ TEST_P(CollectionsEraserTest, MB_50747_delete_and_drop) {
21772177
EXPECT_EQ(0, vb->getNumItems());
21782178
}
21792179

2180+
TEST_P(CollectionsEraserTest, drop_after_modify) {
2181+
// add two collections
2182+
CollectionsManifest cm(CollectionEntry::dairy);
2183+
vb->updateFromManifest(makeManifest(cm.add(CollectionEntry::fruit)));
2184+
2185+
flushVBucketToDiskIfPersistent(vbid, 2 /* 2 x system */);
2186+
2187+
// add some items
2188+
store_item(
2189+
vbid, StoredDocKey{"dairy:milk", CollectionEntry::dairy}, "nice");
2190+
store_item(vbid,
2191+
StoredDocKey{"dairy:butter", CollectionEntry::dairy},
2192+
"lovely");
2193+
store_item(
2194+
vbid, StoredDocKey{"fruit:apple", CollectionEntry::fruit}, "nice");
2195+
store_item(vbid,
2196+
StoredDocKey{"fruit:apricot", CollectionEntry::fruit},
2197+
"lovely");
2198+
2199+
flushVBucketToDiskIfPersistent(vbid, 4);
2200+
2201+
EXPECT_EQ(4, vb->getNumItems());
2202+
2203+
// Modify history setting of dairy false->true
2204+
vb->updateFromManifest(makeManifest(
2205+
cm.update(CollectionEntry::fruit, cb::NoExpiryLimit, true)));
2206+
store_item(
2207+
vbid, StoredDocKey{"fruit:apple", CollectionEntry::fruit}, "nice");
2208+
2209+
flushVBucketToDiskIfPersistent(vbid, 2);
2210+
2211+
// delete the collections
2212+
vb->updateFromManifest(makeManifest(
2213+
cm.remove(CollectionEntry::dairy).remove(CollectionEntry::fruit)));
2214+
2215+
EXPECT_FALSE(vb->lockCollections().exists(CollectionEntry::dairy));
2216+
EXPECT_FALSE(vb->lockCollections().exists(CollectionEntry::fruit));
2217+
2218+
flushVBucketToDiskIfPersistent(vbid, 2 /* 2 x system */);
2219+
2220+
if (!isPersistent()) {
2221+
// 2x create collection, 1x modify
2222+
EXPECT_EQ(3, vb->ht.getNumSystemItems());
2223+
}
2224+
2225+
runCollectionsEraser(vbid);
2226+
2227+
EXPECT_EQ(0, vb->getNumItems());
2228+
2229+
EXPECT_FALSE(vb->lockCollections().exists(CollectionEntry::dairy));
2230+
EXPECT_FALSE(vb->lockCollections().exists(CollectionEntry::fruit));
2231+
2232+
if (!isPersistent()) {
2233+
// 2 system events remain (tombstones of dairy/fruit).
2234+
// modify was purged
2235+
EXPECT_EQ(2, vb->ht.getNumSystemItems());
2236+
}
2237+
}
2238+
21802239
// Test cases which run for persistent and ephemeral buckets
21812240
INSTANTIATE_TEST_SUITE_P(CollectionsEraserTests,
21822241
CollectionsEraserTest,

0 commit comments

Comments
 (0)