Skip to content

refactor(cache): move rwlock from LruEntry to FileStore, to reduce memory usage #1149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EricHuangqx merged 3 commits into from
Mar 20, 2026
+66 −18
Merged

refactor(cache): move rwlock from LruEntry to FileStore, to reduce memory usage #1149

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8eab878
refactor(cache): lazy initialize rwlock to reduce memory usage
xiaoyang-hhh Mar 19, 2026
f073493
fix build
xiaoyang-hhh Mar 19, 2026
b33c5b9
use defer to release
xiaoyang-hhh Mar 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions fs/cache/full_file_cache/cache_pool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,13 @@ int FileCachePool::evict(std::string_view filename) {
lru_.mark_key_cleared(lruEntry->lruIter);
}
int err = 0;
auto cacheStore = static_cast<FileCacheStore*>(open(filePath, O_RDWR, 0644));
{
Copy link
Collaborator

@lihuiba lihuiba Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DEFER(cacheStore->release())

Copy link
Contributor Author

@xiaoyang-hhh xiaoyang-hhh Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

photon::scoped_rwlock rl(lruEntry->rw_lock_, photon::WLOCK);
photon::scoped_rwlock rl(cacheStore->rw_lock(), photon::WLOCK);
err = mediaFs_->truncate(filePath.data(), 0);
lruEntry->truncate_done = false;
}
cacheStore->release();
if (err) {
ERRNO e;
LOG_ERROR("truncate(0) failed, name: `, ret: `, error code: `", filePath,
Expand Down Expand Up @@ -256,11 +258,13 @@ void FileCachePool::eviction() {
continue;
}

auto cacheStore = static_cast<FileCacheStore*>(open(fileName, O_RDWR, 0644));
{
photon::scoped_rwlock rl(lruEntry->rw_lock_, photon::WLOCK);
photon::scoped_rwlock rl(cacheStore->rw_lock(), photon::WLOCK);
err = mediaFs_->truncate(fileName.data(), 0);
lruEntry->truncate_done = false;
}
cacheStore->release();

if (err) {
ERRNO e;
Expand Down
6 changes: 2 additions & 4 deletions fs/cache/full_file_cache/cache_pool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,6 @@ class FileCachePool : public photon::fs::ICachePool {

void Init();

// pathname must begin with '/'
photon::fs::ICacheStore *do_open(std::string_view pathname, int flags, mode_t mode) override;

int set_quota(std::string_view pathname, size_t quota) override;
int stat(photon::fs::CacheStat *stat,
std::string_view pathname = std::string_view(nullptr, 0)) override;
Expand All @@ -64,7 +61,6 @@ class FileCachePool : public photon::fs::ICachePool {
uint32_t lruIter;
int openCount;
uint64_t size;
photon::rwlock rw_lock_;
bool truncate_done;
};

Expand All @@ -81,6 +77,8 @@ class FileCachePool : public photon::fs::ICachePool {
uint64_t updateSpace(FileNameMap::iterator iter, uint64_t size);

protected:
// pathname must begin with '/'
photon::fs::ICacheStore *do_open(std::string_view pathname, int flags, mode_t mode) override;
photon::fs::IFile *openMedia(std::string_view name, int flags, int mode);

static uint64_t timerHandler(void *data);
Expand Down
7 changes: 3 additions & 4 deletions fs/cache/full_file_cache/cache_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ FileCacheStore::~FileCacheStore() {

ICacheStore::try_preadv_result FileCacheStore::try_preadv2(const struct iovec *iov, int iovcnt,
off_t offset, int flags) {
auto lruEntry = static_cast<FileCachePool::LruEntry *>(iterator_->second.get());
photon::scoped_rwlock rl(lruEntry->rw_lock_, photon::RLOCK);
photon::scoped_rwlock rl(rw_lock_, photon::RLOCK);
return this->ICacheStore::try_preadv2(iov, iovcnt, offset, flags);
}

Expand All @@ -70,8 +69,8 @@ ssize_t FileCacheStore::do_pwritev(const struct iovec *iov, int iovcnt, off_t of
{
ssize_t ret;
iovector_view view((iovec*)iov, iovcnt);
auto lruEntry = static_cast<FileCachePool::LruEntry *>(iterator_->second.get());
photon::scoped_rwlock rl(lruEntry->rw_lock_, photon::RLOCK);
auto lruEntry = iterator_->second.get();
photon::scoped_rwlock rl(rw_lock_, photon::RLOCK);
if (!lruEntry->truncate_done) {
// May repeated ftruncate() here, but it doesn't matter
ret = localFile_->ftruncate(actual_size_);
Expand Down
4 changes: 4 additions & 0 deletions fs/cache/full_file_cache/cache_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ class FileCacheStore : public photon::fs::ICacheStore {

int fstat(struct stat *buf) override;

photon::rwlock &rw_lock() { return rw_lock_; }

protected:
bool cacheIsFull();

Expand All @@ -70,6 +72,8 @@ class FileCacheStore : public photon::fs::ICacheStore {
FileIterator iterator_;
RangeLock rangeLock_;

photon::rwlock rw_lock_;

ssize_t do_pwritev(const struct iovec *iov, int iovcnt, off_t offset);
};

Expand Down
11 changes: 9 additions & 2 deletions fs/cache/full_file_cache/quota_pool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,10 @@ int QuotaFilePool::evict(std::string_view filename) {
const auto& filePath = fileIter->first;
int err;
auto lruEntry = static_cast<QuotaLruEntry*>(fileIter->second.get());

auto cacheStore = static_cast<FileCacheStore*>(open(filePath, O_RDWR, 0644));
Copy link
Collaborator

@lihuiba lihuiba Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DEFER(cacheStore->release())

Copy link
Contributor Author

@xiaoyang-hhh xiaoyang-hhh Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

{
photon::scoped_rwlock rl(lruEntry->rw_lock_, photon::WLOCK);
photon::scoped_rwlock rl(cacheStore->rw_lock(), photon::WLOCK);
lru.mark_key_cleared(lruEntry->QuotaLruIter);
err = mediaFs_->truncate(filePath.data(), 0);
if (err) {
Expand All @@ -207,6 +209,7 @@ int QuotaFilePool::evict(std::string_view filename) {
}
afterFtrucate(fileIter);
}
cacheStore->release();
photon::thread_yield();
return 0;
}
Expand Down Expand Up @@ -234,15 +237,19 @@ void QuotaFilePool::dirEviction() {
auto lruEntry = static_cast<QuotaLruEntry*>(fileIter->second.get());
int err;
bool flags_dir_delete = false;

auto cacheStore = static_cast<FileCacheStore*>(open(fileName, O_RDWR, 0644));
{
Copy link
Collaborator

@lihuiba lihuiba Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DEFER(cacheStore->release())

Copy link
Contributor Author

@xiaoyang-hhh xiaoyang-hhh Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

photon::scoped_rwlock rl(lruEntry->rw_lock_, photon::WLOCK);
photon::scoped_rwlock rl(cacheStore->rw_lock(), photon::WLOCK);
if (lruEntry->openCount==0){
dir->lru.mark_key_cleared(lruEntry->QuotaLruIter);
} else {
dir->lru.access(lruEntry->QuotaLruIter);
}
err = mediaFs_->truncate(fileName.data(), 0);
}
cacheStore->release();

if (err) {
ERRNO e;
LOG_ERROR("truncate(0) failed, name : `, ret : `, error code : `", fileName, err, e);
Expand Down
3 changes: 1 addition & 2 deletions fs/cache/full_file_cache/quota_pool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ class QuotaFilePool : public FileCachePool {
QuotaFilePool(photon::fs::IFileSystem* mediaFs, uint64_t capacityInGB, uint64_t periodInUs,
uint64_t diskAvailInBytes, uint64_t refillUnit, int quotaDirLevel);

photon::fs::ICacheStore* do_open(std::string_view pathname, int flags, mode_t mode) override;

void updateDirLru(FileIterator iter);

bool dirSpaceIsFull(FileIterator iter);
Expand All @@ -42,6 +40,7 @@ class QuotaFilePool : public FileCachePool {
int evict(std::string_view filename) override;

protected:
photon::fs::ICacheStore* do_open(std::string_view pathname, int flags, mode_t mode) override;
void dirEviction();

bool afterFtrucate(FileIterator iter) override;
Expand Down
3 changes: 1 addition & 2 deletions fs/cache/full_file_cache/quota_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,7 @@ ssize_t QuotaFileStore::do_pwritev2(const struct iovec *iov, int iovcnt, off_t o
}
ssize_t ret;
{
auto lruEntry = static_cast<FileCachePool::LruEntry*>(iterator_->second.get());
photon::scoped_rwlock wl(lruEntry->rw_lock_, photon::WLOCK);
photon::scoped_rwlock wl(rw_lock_, photon::WLOCK);
ret = localFile_->pwritev(iov, iovcnt, offset);
}
if (ret < 0 && ENOSPC == errno) {
Expand Down
4 changes: 2 additions & 2 deletions fs/cache/pool_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,6 @@ namespace fs

void set_trans_func(CacheFnTransFunc fn_trans_func);

virtual ICacheStore* do_open(std::string_view filename, int flags, mode_t mode) = 0;

virtual int rename(std::string_view oldname, std::string_view newname) = 0;

virtual ssize_t list(const char* dirname, ListType type,
Expand Down Expand Up @@ -146,6 +144,8 @@ namespace fs
}

protected:
virtual ICacheStore* do_open(std::string_view filename, int flags, mode_t mode) = 0;

void* m_stores;
CacheFnTransFunc fn_trans_func;
void* m_thread_pool = nullptr;
Expand Down
38 changes: 38 additions & 0 deletions fs/cache/test/cache_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,44 @@ TEST(CachePool, random_evict_file) {
if (cacheStore) cacheStore->release();
}

TEST(CachePool, open_same_file) {
std::string root = "/tmp/ease/cache/open_same_file/";
SetupTestDir(root);
auto mediaFs = new_localfs_adaptor(root.c_str(), ioengine_libaio);
auto alignFs = new_aligned_fs_adaptor(mediaFs, 4 * 1024, true, true);
auto cacheAllocator = new AlignedAlloc(4 * 1024);
auto roCachedFs = new_full_file_cached_fs(nullptr, alignFs, 1024 * 1024,
1, 1000 * 1000 * 1, 128ul * 1024 * 1024, cacheAllocator, 0);
auto cachePool = roCachedFs->get_pool();
DEFER({ delete cacheAllocator; delete roCachedFs; });

auto fileName = "/testDir/testfile";

uint64_t written = 0;
for (int j = 0; j < 5; j++) {
std::vector<ICacheStore*> cacheStores;
for (int i = 0; i < 10; i++) {
cacheStores.push_back(cachePool->open(fileName, O_CREAT | O_RDWR, 0644));
ASSERT_NE(nullptr, cacheStores.back());
}

const size_t bufSize = 1024 * 1024;
IOVector buffer(*cacheAllocator);
buffer.push_back(bufSize);

for (auto cacheStore : cacheStores) {
if (rand() % 2 == 0) {
cacheStore->release();
} else {
auto ret = cacheStore->do_pwritev2(buffer.iovec(), buffer.iovcnt(), written, 0);
EXPECT_EQ(ret, (ssize_t)bufSize);
written += ret;
cacheStore->release();
}
}
}
}

}
}
int main(int argc, char** argv) {
Expand Down
Loading