Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions cloud/filestore/libs/storage/tablet/tablet_actor_listnodes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -139,11 +139,16 @@ bool TIndexTabletActor::PrepareTx_ListNodes(
// TODO: AccessCheck
TABLET_VERIFY(args.Node);

ui32 maxRows = Max<ui32>();
if (args.Request.GetMaxRows() > 0) {
maxRows = args.Request.GetMaxRows();
}

if (!PrechargeNodeRefs(
db,
args.NodeId,
args.Cookie,
Max<ui64>(),
maxRows,
args.BytesToPrecharge))
{
return false; // not ready
Expand All @@ -160,7 +165,8 @@ bool TIndexTabletActor::PrepareTx_ListNodes(
args.MaxBytes,
&args.Next,
Config->GetNodeRefsNoAutoPrecharge(),
args.Request.GetListNodesSizeMode()))
args.Request.GetListNodesSizeMode(),
maxRows))
{
ready = false;
}
Expand Down
27 changes: 18 additions & 9 deletions cloud/filestore/libs/storage/tablet/tablet_database.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,8 @@ bool TIndexTabletDatabase::ReadNodeRefsBase(
ui32 maxBytes,
TString* next,
ui32* skippedRefs,
NProto::EListNodesSizeMode sizeMode)
NProto::EListNodesSizeMode sizeMode,
ui32 maxRows)
{

using TTableBase = typename TIndexTabletSchema::NodeRefs;
Expand Down Expand Up @@ -623,6 +624,7 @@ bool TIndexTabletDatabase::ReadNodeRefsBase(
} else {
bytes += ref.Name.size();
}
--maxRows;
} else {
++skipped;
}
Expand All @@ -631,7 +633,7 @@ bool TIndexTabletDatabase::ReadNodeRefsBase(
return false; // not ready
}

if (maxBytes && bytes >= maxBytes) {
if (maxBytes && bytes >= maxBytes || maxRows == 0) {
break;
}
}
Expand All @@ -656,7 +658,8 @@ TIndexTabletDatabase::ReadNodeRefsBase<TIndexTabletSchema::NodeRefs>(
ui32,
TString*,
ui32*,
NProto::EListNodesSizeMode);
NProto::EListNodesSizeMode,
ui32 maxRows);

template bool
TIndexTabletDatabase::ReadNodeRefsBase<TIndexTabletSchema::NodeRefsNoPrecharge>(
Expand All @@ -667,7 +670,8 @@ TIndexTabletDatabase::ReadNodeRefsBase<TIndexTabletSchema::NodeRefsNoPrecharge>(
ui32,
TString*,
ui32*,
NProto::EListNodesSizeMode);
NProto::EListNodesSizeMode,
ui32 maxRows);

bool TIndexTabletDatabase::ReadNodeRefs(
ui64 nodeId,
Expand All @@ -678,7 +682,8 @@ bool TIndexTabletDatabase::ReadNodeRefs(
TString* next,
ui32* skippedRefs,
bool noAutoPrecharge,
NProto::EListNodesSizeMode sizeMode)
NProto::EListNodesSizeMode sizeMode,
ui32 maxRows)
{
if (noAutoPrecharge) {
return ReadNodeRefsBase<TIndexTabletSchema::NodeRefsNoPrecharge>(
Expand All @@ -689,7 +694,8 @@ bool TIndexTabletDatabase::ReadNodeRefs(
maxBytes,
next,
skippedRefs,
sizeMode);
sizeMode,
maxRows);
}
return ReadNodeRefsBase<TIndexTabletSchema::NodeRefs>(
nodeId,
Expand All @@ -699,7 +705,8 @@ bool TIndexTabletDatabase::ReadNodeRefs(
maxBytes,
next,
skippedRefs,
sizeMode);
sizeMode,
maxRows);
}

bool TIndexTabletDatabase::ReadNodeRefs(
Expand Down Expand Up @@ -2243,7 +2250,8 @@ bool TIndexTabletDatabaseProxy::ReadNodeRefs(
TString* next,
ui32* skippedRefs,
bool noAutoPrecharge,
NProto::EListNodesSizeMode sizeMode)
NProto::EListNodesSizeMode sizeMode,
ui32 maxRows)
{
ui32 skipped = 0;
if (!skippedRefs) {
Expand All @@ -2258,7 +2266,8 @@ bool TIndexTabletDatabaseProxy::ReadNodeRefs(
next,
skippedRefs,
noAutoPrecharge,
sizeMode);
sizeMode,
maxRows);
if (result) {
// If ReadNodeRefs was successful, it is reasonable to update the cache
// with the values that have just been read.
Expand Down
9 changes: 6 additions & 3 deletions cloud/filestore/libs/storage/tablet/tablet_database.h
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,8 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_STATS)
TString* next,
ui32* skippedRefs,
bool noAutoPrecharge,
NProto::EListNodesSizeMode sizeMode) override;
NProto::EListNodesSizeMode sizeMode,
ui32 maxRows) override;

private:
template <typename TTable>
Expand All @@ -226,7 +227,8 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_STATS)
ui32 maxBytes,
TString* next,
ui32* skippedRefs,
NProto::EListNodesSizeMode sizeMode);
NProto::EListNodesSizeMode sizeMode,
ui32 maxRows);

public:

Expand Down Expand Up @@ -651,7 +653,8 @@ class TIndexTabletDatabaseProxy: public TIndexTabletDatabase
TString* next,
ui32* skippedRefs,
bool noAutoPrecharge,
NProto::EListNodesSizeMode) override;
NProto::EListNodesSizeMode,
ui32 maxRows) override;

bool ReadNodeRefs(
ui64 startNodeId,
Expand Down
6 changes: 4 additions & 2 deletions cloud/filestore/libs/storage/tablet/tablet_database_ut.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -598,7 +598,8 @@ Y_UNIT_TEST_SUITE(TIndexTabletDatabaseTest)
nullptr,
nullptr,
false,
NProto::LNSM_NAME_ONLY));
NProto::LNSM_NAME_ONLY,
Max<ui32>()));
UNIT_ASSERT_VALUES_EQUAL(2, refs.size());
});

Expand All @@ -615,7 +616,8 @@ Y_UNIT_TEST_SUITE(TIndexTabletDatabaseTest)
nullptr,
nullptr,
true,
NProto::LNSM_NAME_ONLY));
NProto::LNSM_NAME_ONLY,
Max<ui32>()));
UNIT_ASSERT_VALUES_EQUAL(2, refs.size());
});
}
Expand Down
3 changes: 2 additions & 1 deletion cloud/filestore/libs/storage/tablet/tablet_state.h
Original file line number Diff line number Diff line change
Expand Up @@ -634,7 +634,8 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_COUNTER)
ui32 maxBytes,
TString* next = nullptr,
bool noAutoPrecharge = false,
NProto::EListNodesSizeMode sizeMode = NProto::LNSM_NAME_ONLY);
NProto::EListNodesSizeMode sizeMode = NProto::LNSM_NAME_ONLY,
ui32 maxRows = Max<ui32>());

bool ReadNodeRefs(
IIndexTabletDatabase& db,
Expand Down
10 changes: 8 additions & 2 deletions cloud/filestore/libs/storage/tablet/tablet_state_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,8 @@ bool TInMemoryIndexState::ReadNodeRefs(
TString* next,
ui32* skippedRefs,
bool noAutoPrecharge,
NProto::EListNodesSizeMode sizeMode)
NProto::EListNodesSizeMode sizeMode,
ui32 maxRows)
{
Y_UNUSED(noAutoPrecharge); // Not applicable to in-memory cache
if (!NodeRefsExhaustivenessInfo.IsExhaustiveForNode(nodeId)) {
Expand All @@ -278,6 +279,10 @@ bool TInMemoryIndexState::ReadNodeRefs(

auto it = NodeRefs.lower_bound(TNodeRefsKey(nodeId, cookie));

if (maxRows == 0) {
maxRows = Max<ui32>();
}

ui32 bytes = 0;
ui32 skipped = 0;
while (it != NodeRefs.end() && it->first.NodeId == nodeId) {
Expand All @@ -303,13 +308,14 @@ bool TInMemoryIndexState::ReadNodeRefs(
} else {
bytes += ref.Name.size();
}
--maxRows;
} else {
++skipped;
}

++it;

if (maxBytes && bytes >= maxBytes) {
if (maxBytes && bytes >= maxBytes || maxRows == 0) {
break;
}
}
Expand Down
4 changes: 3 additions & 1 deletion cloud/filestore/libs/storage/tablet/tablet_state_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,9 @@ class TInMemoryIndexState : public IIndexTabletDatabase
TString* next,
ui32* skippedRefs,
bool noAutoPrecharge,
NProto::EListNodesSizeMode sizeMode = NProto::LNSM_NAME_ONLY) override;
NProto::EListNodesSizeMode sizeMode = NProto::LNSM_NAME_ONLY,
ui32 maxRows = Max<ui32>()) override;


bool ReadNodeRefs(
ui64 startNodeId,
Expand Down
4 changes: 3 additions & 1 deletion cloud/filestore/libs/storage/tablet/tablet_state_iface.h
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,9 @@ class IIndexTabletDatabase
TString* next = nullptr,
ui32* skippedRefs = nullptr,
bool noAutoPrecharge = false,
NProto::EListNodesSizeMode sizeMode = NProto::LNSM_NAME_ONLY) = 0;
NProto::EListNodesSizeMode sizeMode = NProto::LNSM_NAME_ONLY,
ui32 maxRows = Max<ui32>()) = 0;


/**
* @brief read at most maxCount node refs starting from key
Expand Down
7 changes: 5 additions & 2 deletions cloud/filestore/libs/storage/tablet/tablet_state_nodes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,8 @@ bool TIndexTabletState::ReadNodeRefs(
ui32 maxBytes,
TString* next,
bool noAutoPrecharge,
NProto::EListNodesSizeMode sizeMode)
NProto::EListNodesSizeMode sizeMode,
ui32 maxRows)
{
bool ready = db.ReadNodeRefs(
nodeId,
Expand All @@ -524,11 +525,13 @@ bool TIndexTabletState::ReadNodeRefs(
next,
nullptr, // skippedRefs
noAutoPrecharge,
sizeMode);
sizeMode,
maxRows);

ui64 checkpointId = Impl->Checkpoints.FindCheckpoint(nodeId, commitId);
if (checkpointId != InvalidCommitId) {
// there could be history versions
// maxBytes and maxCount are not respected here.
if (!db.ReadNodeRefVers(nodeId, commitId, refs)) {
ready = false;
}
Expand Down
13 changes: 13 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_ut_nodes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,19 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Nodes)

UNIT_ASSERT(response->Record.GetCookie().empty());
}

// Limit by elements count.
{
auto response = tablet.ListNodes(RootNodeId, 0, TString{}, 1);
const auto& names = response->Record.GetNames();
UNIT_ASSERT_VALUES_EQUAL(1, names.size());
UNIT_ASSERT_VALUES_EQUAL("test1", names[0]);
const auto& nodes = response->Record.GetNodes();
UNIT_ASSERT_VALUES_EQUAL(1, nodes.size());
UNIT_ASSERT_VALUES_EQUAL(id1, nodes.Get(0).GetId());

UNIT_ASSERT_VALUES_EQUAL("test2", response->Record.GetCookie());
}
}

Y_UNIT_TEST(ShouldLimitListNodesWithFullRowSizeMode)
Expand Down
10 changes: 10 additions & 0 deletions cloud/filestore/libs/storage/testlib/tablet_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,16 @@ class TIndexTabletClient
return request;
}

auto CreateListNodesRequest(ui64 node, ui32 bytes, TString cookie, ui32 maxRows)
{
auto request = CreateSessionRequest<TEvService::TEvListNodesRequest>();
request->Record.SetNodeId(node);
request->Record.SetMaxBytes(bytes);
request->Record.SetCookie(std::move(cookie));
request->Record.SetMaxRows(maxRows);
return request;
}

auto CreateReadNodeRefsRequest(ui64 node, TString cookie, ui32 limit){
auto request = std::make_unique<TEvIndexTablet::TEvReadNodeRefsRequest>();
request->Record.SetNodeId(node);
Expand Down
3 changes: 3 additions & 0 deletions cloud/filestore/public/api/protos/node.proto
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,9 @@ message TListNodesRequest

// Mode for calculating byte size when applying MaxBytes limit.
EListNodesSizeMode ListNodesSizeMode = 7;

// Optional limit for response nodes count.
uint32 MaxRows = 8;
}

message TListNodesResponse
Expand Down
Loading