Skip to content

Commit a278a37

Browse files
UESTC-AHaoUESTC-AHao
committed
3FS
1 parent 5e066d6 commit a278a37

19 files changed

+1647
-0
lines changed

‎ucm/store/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ target_link_libraries(storeintf INTERFACE storedetail infra_status)
55
add_subdirectory(nfsstore)
66
add_subdirectory(pcstore)
77
add_subdirectory(posix)
8+
add_subdirectory(ds3fs)
89
add_subdirectory(cache)
910
add_subdirectory(mooncakestore)
1011
add_subdirectory(test)

‎ucm/store/ds3fs/CMakeLists.txt‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
find_path(HF3FS_USRBIO_INCLUDE_DIR NAMES hf3fs_usrbio.h PATHS /usr/local/3fs/src/lib/api NO_DEFAULT_PATH)
2+
find_library(HF3FS_USRBIO_LIBRARY NAMES hf3fs_api_shared PATHS /usr/local/3fs/src/lib/rs/hf3fs-usrbio-sys/lib NO_DEFAULT_PATH)
3+
4+
if(HF3FS_USRBIO_INCLUDE_DIR AND HF3FS_USRBIO_LIBRARY)
5+
file(GLOB_RECURSE UCM_DS3FS_STORE_CC_SOURCE_FILES "./cc/*.cc")
6+
add_library(ds3fsstore STATIC ${UCM_DS3FS_STORE_CC_SOURCE_FILES})
7+
target_include_directories(ds3fsstore PUBLIC
8+
${CMAKE_CURRENT_SOURCE_DIR}/cc
9+
)
10+
11+
target_include_directories(ds3fsstore PRIVATE ${HF3FS_USRBIO_INCLUDE_DIR})
12+
target_link_libraries(ds3fsstore PRIVATE ${HF3FS_USRBIO_LIBRARY})
13+
target_link_libraries(ds3fsstore PUBLIC storeintf infra_logger)
14+
15+
file(GLOB_RECURSE UCM_DS3FS_STORE_CPY_SOURCE_FILES "./cpy/*.cc")
16+
pybind11_add_module(ucmds3fsstore ${UCM_DS3FS_STORE_CPY_SOURCE_FILES})
17+
target_link_libraries(ucmds3fsstore PRIVATE ds3fsstore)
18+
19+
file(RELATIVE_PATH INSTALL_REL_PATH ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
20+
install(TARGETS ucmds3fsstore LIBRARY DESTINATION ${INSTALL_REL_PATH} COMPONENT ucm)
21+
else()
22+
message(STATUS "ds3fsstore: Skipping build - required HF3FS dependencies not found")
23+
if(NOT HF3FS_USRBIO_INCLUDE_DIR)
24+
message(STATUS " - Missing: hf3fs_usrbio.h ")
25+
endif()
26+
if(NOT HF3FS_USRBIO_LIBRARY)
27+
message(STATUS " - Missing: hf3fs_api_shared library ")
28+
endif()
29+
message(STATUS " Please ensure HF3FS dependencies are installed or build paths are correct")
30+
endif()

‎ucm/store/ds3fs/cc/ds3fs_file.cc‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/**
2+
* MIT License
3+
*
4+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
* */
24+
#include "ds3fs_file.h"
25+
#include <sys/stat.h>
26+
#include <unistd.h>
27+
28+
namespace UC::Ds3fsStore {
29+
30+
static constexpr auto NewFilePerm = (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
31+
static constexpr auto NewDirPerm = (S_IRWXU | S_IRGRP | S_IROTH);
32+
33+
Ds3fsFile::~Ds3fsFile()
34+
{
35+
if (handle_ != -1) { Close(); }
36+
}
37+
38+
Status Ds3fsFile::MkDir()
39+
{
40+
auto ret = mkdir(path_.c_str(), NewDirPerm);
41+
auto eno = errno;
42+
if (ret != 0) [[unlikely]] {
43+
if (eno == EEXIST) { return Status::DuplicateKey(); }
44+
return Status::OsApiError(std::to_string(eno));
45+
}
46+
return Status::OK();
47+
}
48+
49+
Status Ds3fsFile::RmDir()
50+
{
51+
auto ret = rmdir(path_.c_str());
52+
auto eno = errno;
53+
if (ret != 0) [[unlikely]] { return Status::OsApiError(std::to_string(eno)); }
54+
return Status::OK();
55+
}
56+
57+
Status Ds3fsFile::Rename(const std::string& newName)
58+
{
59+
auto ret = rename(path_.c_str(), newName.c_str());
60+
auto eno = errno;
61+
if (ret != 0) [[unlikely]] {
62+
if (eno == ENOENT) { return Status::NotFound(); }
63+
return Status::OsApiError(std::to_string(eno));
64+
}
65+
return Status::OK();
66+
}
67+
68+
Status Ds3fsFile::Access(const int32_t mode)
69+
{
70+
auto ret = access(path_.c_str(), mode);
71+
auto eno = errno;
72+
if (ret != 0) [[unlikely]] {
73+
if (eno == ENOENT) { return Status::NotFound(); }
74+
return Status::OsApiError(std::to_string(eno));
75+
}
76+
return Status::OK();
77+
}
78+
79+
Status Ds3fsFile::Open(const uint32_t flags)
80+
{
81+
handle_ = open(path_.c_str(), flags, NewFilePerm);
82+
auto eno = errno;
83+
if (handle_ < 0) [[unlikely]] {
84+
if (eno == EEXIST) { return Status::DuplicateKey(); }
85+
return Status::OsApiError(std::to_string(eno));
86+
}
87+
return Status::OK();
88+
}
89+
90+
void Ds3fsFile::Close()
91+
{
92+
close(handle_);
93+
handle_ = -1;
94+
}
95+
96+
void Ds3fsFile::Remove() { remove(path_.c_str()); }
97+
98+
} // namespace UC::Ds3fsStore

‎ucm/store/ds3fs/cc/ds3fs_file.h‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/**
2+
* MIT License
3+
*
4+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
* */
24+
#ifndef UNIFIEDCACHE_DS3FS_STORE_CC_DS3FS_FILE_H
25+
#define UNIFIEDCACHE_DS3FS_STORE_CC_DS3FS_FILE_H
26+
27+
#include <fcntl.h>
28+
#include "status/status.h"
29+
30+
namespace UC::Ds3fsStore {
31+
32+
class Ds3fsFile {
33+
public:
34+
struct AccessMode {
35+
static constexpr int32_t READ = R_OK;
36+
static constexpr int32_t WRITE = W_OK;
37+
static constexpr int32_t EXIST = F_OK;
38+
static constexpr int32_t EXECUTE = X_OK;
39+
};
40+
struct OpenFlag {
41+
static constexpr uint32_t READ_ONLY = O_RDONLY;
42+
static constexpr uint32_t WRITE_ONLY = O_WRONLY;
43+
static constexpr uint32_t READ_WRITE = O_RDWR;
44+
static constexpr uint32_t CREATE = O_CREAT;
45+
static constexpr uint32_t DIRECT = O_DIRECT;
46+
static constexpr uint32_t APPEND = O_APPEND;
47+
static constexpr uint32_t EXCL = O_EXCL;
48+
};
49+
50+
private:
51+
std::string path_{};
52+
int32_t handle_{-1};
53+
54+
public:
55+
explicit Ds3fsFile(std::string path) : path_{std::move(path)} {}
56+
~Ds3fsFile();
57+
const std::string& Path() const { return path_; }
58+
int32_t ReleaseHandle()
59+
{
60+
int32_t fd = handle_;
61+
handle_ = -1;
62+
return fd;
63+
}
64+
Status MkDir();
65+
Status RmDir();
66+
Status Rename(const std::string& newName);
67+
Status Access(const int32_t mode);
68+
Status Open(const uint32_t flags);
69+
void Close();
70+
void Remove();
71+
};
72+
73+
} // namespace UC::Ds3fsStore
74+
75+
#endif

‎ucm/store/ds3fs/cc/ds3fs_store.cc‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/**
2+
* MIT License
3+
*
4+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
* */
24+
#include "ds3fs_store.h"
25+
#include <fmt/ranges.h>
26+
#include "logger/logger.h"
27+
#include "space_manager.h"
28+
#include "trans_manager.h"
29+
30+
namespace UC::Ds3fsStore {
31+
32+
class Ds3fsStoreImpl {
33+
public:
34+
SpaceManager spaceMgr;
35+
TransManager transMgr;
36+
bool transEnable{false};
37+
38+
public:
39+
Status Setup(const Config& config)
40+
{
41+
auto s = CheckConfig(config);
42+
if (s.Failure()) [[unlikely]] {
43+
UC_ERROR("Failed to check config params: {}.", s);
44+
return s;
45+
}
46+
s = spaceMgr.Setup(config);
47+
if (s.Failure()) [[unlikely]] { return s; }
48+
transEnable = config.deviceId >= 0;
49+
if (transEnable) {
50+
s = transMgr.Setup(config, spaceMgr.GetLayout());
51+
if (s.Failure()) [[unlikely]] { return s; }
52+
}
53+
ShowConfig(config);
54+
return Status::OK();
55+
}
56+
57+
private:
58+
Status CheckConfig(const Config& config)
59+
{
60+
if (config.storageBackends.empty()) {
61+
return Status::InvalidParam("invalid storage backends");
62+
}
63+
if (config.deviceId < -1) {
64+
return Status::InvalidParam("invalid device({})", config.deviceId);
65+
}
66+
if (config.deviceId == -1) { return Status::OK(); }
67+
if (config.tensorSize == 0 || config.shardSize < config.tensorSize ||
68+
config.blockSize < config.shardSize || config.shardSize % config.tensorSize != 0 ||
69+
config.blockSize % config.shardSize != 0) {
70+
return Status::InvalidParam("invalid size({},{},{})", config.tensorSize,
71+
config.shardSize, config.blockSize);
72+
}
73+
if (config.streamNumber == 0) {
74+
return Status::InvalidParam("invalid stream number({})", config.streamNumber);
75+
}
76+
return Status::OK();
77+
}
78+
void ShowConfig(const Config& config)
79+
{
80+
constexpr const char* ns = "Ds3fsStore";
81+
std::string buildType = UCM_BUILD_TYPE;
82+
if (buildType.empty()) { buildType = "Release"; }
83+
UC_INFO("{}-{}({}).", ns, UCM_COMMIT_ID, buildType);
84+
UC_INFO("Set {}::StorageBackends to {}.", ns, config.storageBackends[0]);
85+
UC_INFO("Set {}::DeviceId to {}.", ns, config.deviceId);
86+
if (config.deviceId == -1) { return; }
87+
UC_INFO("Set {}::TensorSize to {}.", ns, config.tensorSize);
88+
UC_INFO("Set {}::ShardSize to {}.", ns, config.shardSize);
89+
UC_INFO("Set {}::BlockSize to {}.", ns, config.blockSize);
90+
UC_INFO("Set {}::IoDirect to {}.", ns, config.ioDirect);
91+
UC_INFO("Set {}::StreamNumber to {}.", ns, config.streamNumber);
92+
UC_INFO("Set {}::TimeoutMs to {}.", ns, config.timeoutMs);
93+
}
94+
};
95+
96+
Ds3fsStore::~Ds3fsStore() = default;
97+
98+
Status Ds3fsStore::Setup(const Config& config)
99+
{
100+
try {
101+
impl_ = std::make_shared<Ds3fsStoreImpl>();
102+
} catch (const std::exception& e) {
103+
UC_ERROR("Failed({}) to make ds3fs store object.", e.what());
104+
return Status::Error(e.what());
105+
}
106+
return impl_->Setup(config);
107+
}
108+
109+
std::string Ds3fsStore::Readme() const { return "Ds3fsStore"; }
110+
111+
Expected<std::vector<uint8_t>> Ds3fsStore::Lookup(const Detail::BlockId* blocks, size_t num)
112+
{
113+
return impl_->spaceMgr.Lookup(blocks, num);
114+
}
115+
116+
void Ds3fsStore::Prefetch(const Detail::BlockId* blocks, size_t num) {}
117+
118+
Expected<Detail::TaskHandle> Ds3fsStore::Load(Detail::TaskDesc task)
119+
{
120+
if (!impl_->transEnable) { return Status::Error("transfer is not enable"); }
121+
auto res = impl_->transMgr.Submit({TransTask::Type::LOAD, std::move(task)});
122+
if (!res) [[unlikely]] {
123+
UC_ERROR("Failed({}) to submit load task({}).", res.Error(), task.brief);
124+
}
125+
return res;
126+
}
127+
128+
Expected<Detail::TaskHandle> Ds3fsStore::Dump(Detail::TaskDesc task)
129+
{
130+
if (!impl_->transEnable) { return Status::Error("transfer is not enable"); }
131+
auto res = impl_->transMgr.Submit({TransTask::Type::DUMP, std::move(task)});
132+
if (!res) [[unlikely]] {
133+
UC_ERROR("Failed({}) to submit dump task({}).", res.Error(), task.brief);
134+
}
135+
return res;
136+
}
137+
138+
Expected<bool> Ds3fsStore::Check(Detail::TaskHandle taskId)
139+
{
140+
auto res = impl_->transMgr.Check(taskId);
141+
if (!res) [[unlikely]] { UC_ERROR("Failed({}) to check task({}).", res.Error(), taskId); }
142+
return res;
143+
}
144+
145+
Status Ds3fsStore::Wait(Detail::TaskHandle taskId)
146+
{
147+
auto s = impl_->transMgr.Wait(taskId);
148+
if (s.Failure()) [[unlikely]] { UC_ERROR("Failed({}) to wait task({}).", s, taskId); }
149+
return s;
150+
}
151+
152+
} // namespace UC::Ds3fsStore

0 commit comments

Comments
 (0)