MB-41857: Increase cache limit on close of old db during compaction

During compaction we call openDbForRead for the old database file and
then manually close the file. Manually closing the file causes us to
not track the closure of the file against the file cache limit. This
causes us to decrease the file cache limit with no way to later increase
it, causing opens to later fail as the cache is full.

Change-Id: I3b795263d44e62865e537bc75bd11cb84e107759
Reviewed-on: http://review.couchbase.org/c/kv_engine/+/137569
Tested-by: Build Bot <[email protected]>
Reviewed-by: Dave Rigby <[email protected]>

Author: BenHuddleston

engines/ep/src/couch-kvstore/couch-kvstore.cc

@@ -1309,8 +1309,12 @@ bool CouchKVStore::compactDBInternal(std::unique_lock<std::mutex>& vbLock,
                        false, // copy with lock
                        hook_ctx->stats.preparesPurged,
                        vbid);
-    // Close the source Database File once compaction is done
-    openResult.fileHandle->getDbHolder().close();
+
+    auto oldRev = openResult.fileHandle->getDbHolder().getFileRev();
+
+    // Close the source database file
+    openResult.fileHandle.reset();
+
     // Close the destination file so we may rename it
     targetDb.close();
 
@@ -1376,7 +1380,7 @@ bool CouchKVStore::compactDBInternal(std::unique_lock<std::mutex>& vbLock,
     updateDbFileMap(vbid, new_rev);
 
     // Removing the stale couch file
-    unlinkCouchFile(vbid, openResult.fileHandle->getDbHolder().getFileRev());
+    unlinkCouchFile(vbid, oldRev);
 
     st.compactHisto.add(std::chrono::duration_cast<std::chrono::microseconds>(
             std::chrono::steady_clock::now() - start));

engines/ep/tests/module_tests/couch-kvstore_test.cc

@@ -2104,3 +2104,16 @@ TEST_F(CouchstoreTest, CacheLimitOpenDbForReadFailure) {
 
     EXPECT_EQ(1, CouchKVStoreFileCache::get().getHandle()->capacity());
 }
+
+TEST_F(CouchstoreTest, CacheLimitBumpedOnOldDbCloseDuringCompaction) {
+    CouchKVStoreFileCache::get().getHandle()->resize(5);
+    CompactionConfig config;
+    auto ctx = std::make_shared<CompactionContext>(config, 0);
+
+    std::mutex mutex;
+    std::unique_lock<std::mutex> lock(mutex);
+    EXPECT_EQ(5, CouchKVStoreFileCache::get().getHandle()->capacity());
+
+    EXPECT_TRUE(kvstore->compactDB(lock, ctx));
+    EXPECT_EQ(5, CouchKVStoreFileCache::get().getHandle()->capacity());
+}