Skip to content

Commit 004a998

Browse files
chuandewchuandew
committed
[feat] Add DingoFS Python SDK
C++ layer: - Add public headers: include/dingofs/{client,meta,status,string_slice,data_buffer}.h - Add sdk/shim/ (BindingClient/BindingConfig) as language-agnostic shim layer - Add sdk/cpp/ with public C++ SDK implementation - Add sdk/python/src/ with nanobind bindings and build config - Add pyproject.toml for pip-installable wheel (scikit-build-core) - Add examples/cpp/ with full C++ usage examples - VFS refactor: restructure shim/access_log, fold smooth-upgrade into signatures Python layer: - Add Client wrapper, DingoFile, ScandirIterator, walk(), tests, examples, and docs for the DingoFS Python SDK
1 parent 036d6bd commit 004a998

File tree

115 files changed

+7853
-916
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

115 files changed

+7853
-916
lines changed

‎CMakeLists.txt‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@
1515
# if compile_commands.json is needed, please enable CMAKE_EXPORT_COMPILE_COMMANDS, of use `bear --append -- make` to do make, it's more recommended to use bear.
1616

1717
cmake_minimum_required(VERSION 3.25 FATAL_ERROR)
18-
project(dingofs C CXX)
18+
project(dingofs VERSION 5.1.0 LANGUAGES C CXX)
1919

20+
option(DINGOFS_BUILD_SDK "Build and install the DingoFS client SDK (fat static library)" OFF)
2021
option(BUILD_UNIT_TESTS "Build unit test" OFF)
2122
option(BUILD_INTEGRATION_TESTS "Build integration test" OFF)
2223
option(USE_TCMALLOC "Use tcmalloc" ON)
@@ -31,6 +32,8 @@ message("CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
3132

3233
message("SYSTEM: ${CMAKE_HOST_SYSTEM_NAME} ${CMAKE_HOST_SYSTEM_PROCESSOR}")
3334

35+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
36+
3437
set(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++")
3538

3639
set(CMAKE_FIND_LIBRARY_SUFFIXES .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
@@ -60,6 +63,14 @@ if(NOT DINGOSDK_INSTALL_PATH)
6063
endif()
6164
message("DINGOSDK_INSTALL_PATH:${DINGOSDK_INSTALL_PATH}")
6265

66+
# dingofs SDK install prefix
67+
# Default: ~/.local/dingofs Override: cmake -DCMAKE_INSTALL_PREFIX=/your/path ..
68+
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
69+
set(CMAKE_INSTALL_PREFIX "$ENV{HOME}/.local/dingofs"
70+
CACHE PATH "Install prefix for dingofs SDK (default: ~/.local/dingofs)" FORCE)
71+
endif()
72+
message("CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
73+
6374
set(CMAKE_PREFIX_PATH
6475
${THIRD_PARTY_INSTALL_PATH}
6576
${DINGOSDK_INSTALL_PATH}
@@ -252,12 +263,17 @@ message("Using rados: ${RADOS_LIBRARIES}")
252263
find_package(dingosdk REQUIRED)
253264
message(STATUS "Found dingosdk: ${dingosdk_VERSION}")
254265

255-
find_package(tikvcpp REQUIRED)
256-
message(STATUS "Found tikvcpp: ${tikvcpp_VERSION}")
266+
option(WITH_TIKVCPP "Find and link against tikvcpp (TiKV client)" ON)
267+
if(WITH_TIKVCPP)
268+
find_package(tikvcpp REQUIRED)
269+
message(STATUS "Found tikvcpp: ${tikvcpp_VERSION}")
270+
add_compile_definitions(WITH_TIKVCPP)
271+
endif()
257272

258273
# ------------------------ related to source code ------------------------
259274

260275
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
276+
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
261277

262278
set(PROTO_OUTPUT_DIR ${CMAKE_BINARY_DIR}/protos)
263279

@@ -324,19 +340,23 @@ set_target_properties(PROTO_OBJS
324340
POSITION_INDEPENDENT_CODE ON
325341
)
326342

343+
include(GNUInstallDirs)
344+
345+
# When building Python bindings, all static libraries must be compiled with
346+
# -fPIC so they can be linked into the shared extension module.
347+
if(BUILD_PYTHON_BINDINGS)
348+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
349+
endif()
350+
327351
add_subdirectory(src)
352+
add_subdirectory(sdk)
328353

329354
if(BUILD_UNIT_TESTS)
330355
message(STATUS "Build unit test")
331356
add_subdirectory(test/unit)
332357
endif()
333358

334-
335359
if(BUILD_INTEGRATION_TESTS)
336360
message(STATUS "Build integration test")
337361
add_subdirectory(test/integration)
338362
endif()
339-
340-
341-
342-

‎cmake/FindFUSE3.cmake‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,13 @@ endif()
4141

4242
find_path(FUSE3_INCLUDE_DIR
4343
NAMES fuse_lowlevel.h
44+
HINTS ${THIRD_PARTY_INSTALL_PATH}/include
4445
PATH_SUFFIXES fuse3
4546
)
4647

4748
find_library(FUSE3_LIBRARY
4849
NAMES fuse3
50+
HINTS ${THIRD_PARTY_INSTALL_PATH}/lib
4951
)
5052

5153
mark_as_advanced(FUSE3_INCLUDE_DIR FUSE3_LIBRARY)

‎examples/cpp/CMakeLists.txt‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
cmake_minimum_required(VERSION 3.16)
2+
project(dingofs-demo CXX)
3+
4+
set(CMAKE_CXX_STANDARD 17)
5+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
6+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
7+
8+
# ── dingofs SDK ───────────────────────────────────────────────────────────────
9+
# Tell CMake where dingofs is installed.
10+
# Override: cmake -DDINGOFS_INSTALL_DIR=/opt/dingofs ..
11+
if(NOT DINGOFS_INSTALL_DIR)
12+
set(DINGOFS_INSTALL_DIR "$ENV{HOME}/.local/dingofs")
13+
endif()
14+
list(APPEND CMAKE_PREFIX_PATH "${DINGOFS_INSTALL_DIR}")
15+
16+
find_package(dingofs REQUIRED)
17+
18+
# ── helper to register one example executable ─────────────────────────────────
19+
function(add_example name)
20+
add_executable(${name} ${name}.cc)
21+
target_link_libraries(${name} PRIVATE dingofs::client)
22+
endfunction()
23+
24+
# ── examples ─────────────────────────────────────────────────────────────────
25+
add_example(statfs)
26+
add_example(mkdir)
27+
add_example(write)
28+
add_example(read)
29+
add_example(stat)
30+
add_example(mknod)
31+
add_example(link)
32+
add_example(symlink)
33+
add_example(rename)
34+
add_example(xattr)
35+
add_example(ioctl)
36+
add_example(setflags)
37+
add_example(gflags_demo)

‎examples/cpp/common.h‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright (c) 2026 dingodb.com, Inc. All Rights Reserved
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#pragma once
18+
19+
#include <iostream>
20+
21+
#include "dingofs/client.h"
22+
23+
using dingofs::client::DingofsClient;
24+
25+
// ── Check macro ──────────────────────────────────────────────────────────────
26+
27+
#define CHECK_OK(s, msg) \
28+
do { \
29+
if (!(s).ok()) { \
30+
std::cerr << "[FAIL] " << (msg) << ": " << (s).ToString() << "\n"; \
31+
return 1; \
32+
} \
33+
std::cout << "[ OK ] " << (msg) << "\n"; \
34+
} while (0)
35+
36+
// ── Default config ───────────────────────────────────────────────────────────
37+
38+
inline dingofs::client::DingofsConfig MakeConfig() {
39+
dingofs::client::DingofsConfig cfg;
40+
cfg.mds_addrs = "10.232.10.5:8801";
41+
cfg.fs_name = "test03";
42+
cfg.mount_point = "/dingofs/client/mnt/test03";
43+
cfg.metasystem_type = "mds";
44+
cfg.storage_info = "";
45+
return cfg;
46+
}

‎examples/cpp/gflags_demo.cc‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Copyright (c) 2026 dingodb.com, Inc. All Rights Reserved
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include <fcntl.h>
18+
#include <unistd.h>
19+
20+
#include <cstring>
21+
#include <filesystem>
22+
#include <iostream>
23+
#include <string>
24+
25+
#include "common.h"
26+
#include "dingofs/client.h"
27+
28+
int main() {
29+
// PrintOptions: discover all tunable parameters at runtime
30+
std::cout << "=== PrintOptions ===\n";
31+
DingofsClient::PrintOptions();
32+
std::cout << "\n";
33+
34+
// ListOptions: structured list, just print the count here
35+
auto opts = DingofsClient::ListOptions();
36+
std::cout << "ListOptions: " << opts.size() << " options\n\n";
37+
38+
// GetOption: read defaults
39+
std::string val;
40+
DingofsClient::GetOption("log_dir", &val); std::cout << "log_dir = " << val << "\n";
41+
DingofsClient::GetOption("log_level", &val); std::cout << "log_level = " << val << "\n";
42+
DingofsClient::GetOption("vfs_meta_rpc_timeout_ms", &val); std::cout << "vfs_meta_rpc_timeout_ms = " << val << "\n";
43+
DingofsClient::GetOption("vfs_access_logging", &val); std::cout << "vfs_access_logging = " << val << "\n";
44+
45+
// glog flags are also accessible
46+
DingofsClient::GetOption("logtostderr", &val); std::cout << "logtostderr = " << val << "\n";
47+
DingofsClient::GetOption("minloglevel", &val); std::cout << "minloglevel = " << val << "\n";
48+
std::cout << "\n";
49+
50+
// SetOption: change parameters and read back
51+
const std::string log_dir = "/tmp/dingofs-options-demo";
52+
std::filesystem::create_directories(log_dir);
53+
54+
CHECK_OK(DingofsClient::SetOption("log_dir", log_dir), "SetOption log_dir");
55+
CHECK_OK(DingofsClient::SetOption("log_level", "WARNING"), "SetOption log_level");
56+
CHECK_OK(DingofsClient::SetOption("vfs_meta_rpc_timeout_ms", "3000"), "SetOption vfs_meta_rpc_timeout_ms");
57+
CHECK_OK(DingofsClient::SetOption("vfs_meta_rpc_retry_times", "5"), "SetOption vfs_meta_rpc_retry_times");
58+
CHECK_OK(DingofsClient::SetOption("vfs_access_logging", "false"), "SetOption vfs_access_logging");
59+
CHECK_OK(DingofsClient::SetOption("logtostderr", "false"), "SetOption logtostderr");
60+
CHECK_OK(DingofsClient::SetOption("minloglevel", "1"), "SetOption minloglevel");
61+
62+
// Unknown key must return InvalidParam
63+
auto bad = DingofsClient::SetOption("not_exist_flag", "1");
64+
if (!bad.IsInvalidParam()) {
65+
std::cerr << "[FAIL] unknown key should return InvalidParam\n";
66+
return 1;
67+
}
68+
std::cout << "[ OK ] SetOption unknown key -> InvalidParam\n\n";
69+
70+
// Start with the options set above
71+
DingofsClient client;
72+
CHECK_OK(client.Start(MakeConfig()), "Start");
73+
74+
const uint32_t uid = static_cast<uint32_t>(getuid());
75+
const uint32_t gid = static_cast<uint32_t>(getgid());
76+
77+
// Basic file ops to generate log output
78+
dingofs::Attr dir_attr{};
79+
CHECK_OK(client.MkDir(dingofs::kRootIno, "options_demo", uid, gid, 0755, &dir_attr),
80+
"MkDir");
81+
dingofs::Ino dir_ino = dir_attr.ino;
82+
83+
uint64_t fh = 0;
84+
dingofs::Attr file_attr{};
85+
CHECK_OK(client.Create(dir_ino, "hello.txt", uid, gid, 0644, O_RDWR, &fh, &file_attr),
86+
"Create");
87+
dingofs::Ino file_ino = file_attr.ino;
88+
89+
const char* data = "hello options demo";
90+
uint64_t wsize = 0;
91+
CHECK_OK(client.Write(file_ino, data, strlen(data), 0, fh, &wsize), "Write");
92+
CHECK_OK(client.Flush(file_ino, fh), "Flush");
93+
CHECK_OK(client.Release(file_ino, fh), "Release");
94+
95+
CHECK_OK(client.Unlink(dir_ino, "hello.txt"), "Unlink");
96+
CHECK_OK(client.RmDir(dingofs::kRootIno, "options_demo"), "RmDir");
97+
CHECK_OK(client.Stop(), "Stop");
98+
99+
// Verify logs landed in the directory set via SetOption
100+
bool found = false;
101+
for (const auto& e : std::filesystem::directory_iterator(log_dir)) {
102+
std::cout << " " << e.path().filename().string()
103+
<< " (" << e.file_size() << " bytes)\n";
104+
found = true;
105+
}
106+
if (!found) {
107+
std::cerr << "[FAIL] no log files in " << log_dir << "\n";
108+
return 1;
109+
}
110+
std::cout << "[ OK ] logs written to " << log_dir << "\n";
111+
return 0;
112+
}

‎examples/cpp/ioctl.cc‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright (c) 2026 dingodb.com, Inc. All Rights Reserved
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include <fcntl.h>
18+
#include <linux/fs.h>
19+
#include <iostream>
20+
#include <unistd.h>
21+
22+
#include "common.h"
23+
#include "dingofs/meta.h"
24+
25+
int main() {
26+
DingofsClient::SetOption("log_dir", "/tmp/dingofs-demo-log");
27+
DingofsClient::SetOption("log_level", "WARNING");
28+
29+
dingofs::client::DingofsClient client;
30+
CHECK_OK(client.Start(MakeConfig()), "Start");
31+
32+
const uint32_t uid = static_cast<uint32_t>(getuid());
33+
const uint32_t gid = static_cast<uint32_t>(getgid());
34+
35+
// Setup: create dir + file
36+
dingofs::Attr attr{};
37+
CHECK_OK(client.MkDir(dingofs::kRootIno, "demo_dir", uid, gid, 0755, &attr),
38+
"MkDir");
39+
dingofs::Ino dir_ino = attr.ino;
40+
41+
uint64_t fh = 0;
42+
dingofs::Attr file_attr{};
43+
CHECK_OK(client.Create(dir_ino, "hello.txt", uid, gid, 0644, O_RDWR,
44+
&fh, &file_attr),
45+
"Create");
46+
dingofs::Ino file_ino = file_attr.ino;
47+
CHECK_OK(client.Release(file_ino, fh), "Release");
48+
49+
// Open for ioctl
50+
uint64_t ioc_fh = 0;
51+
CHECK_OK(client.Open(file_ino, O_RDWR, &ioc_fh), "Open");
52+
53+
// Ioctl: FS_IOC_GETFLAGS
54+
int flags_val = 0;
55+
auto s = client.Ioctl(file_ino, uid, FS_IOC_GETFLAGS, 0,
56+
nullptr, 0,
57+
reinterpret_cast<char*>(&flags_val), sizeof(flags_val));
58+
if (s.ok()) {
59+
std::cout << "[ OK ] Ioctl FS_IOC_GETFLAGS: 0x" << std::hex << flags_val
60+
<< std::dec << "\n";
61+
} else {
62+
std::cout << "[ -- ] Ioctl not supported: " << s.ToString() << "\n";
63+
}
64+
65+
CHECK_OK(client.Release(file_ino, ioc_fh), "Release");
66+
67+
// Cleanup
68+
CHECK_OK(client.Unlink(dir_ino, "hello.txt"), "Unlink");
69+
CHECK_OK(client.RmDir(dingofs::kRootIno, "demo_dir"), "RmDir");
70+
71+
CHECK_OK(client.Stop(), "Stop");
72+
return 0;
73+
}

0 commit comments

Comments
 (0)