Skip to content

Commit 31ac77e

Browse files
adameatadameat
authored
switch to scheme shard describe to get storage stats (#15290)
1 parent 29d6dd3 commit 31ac77e

File tree

3 files changed

+107
-30
lines changed

3 files changed

+107
-30
lines changed

‎ydb/core/viewer/json_pipe_req.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,17 @@ TString TViewerPipeClient::GetError(const std::unique_ptr<TEvStateStorage::TEvBo
227227
}
228228
}
229229

230+
bool TViewerPipeClient::IsSuccess(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev) {
231+
return ev->GetRecord().GetStatus() == NKikimrScheme::EStatus::StatusSuccess;
232+
}
233+
234+
TString TViewerPipeClient::GetError(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev) {
235+
if (ev->GetRecord().HasReason()) {
236+
return ev->GetRecord().GetReason();
237+
}
238+
return NKikimrScheme::EStatus_Name(ev->GetRecord().GetStatus());
239+
}
240+
230241
void TViewerPipeClient::RequestHiveDomainStats(NNodeWhiteboard::TTabletId hiveId) {
231242
TActorId pipeClient = ConnectTabletPipe(hiveId);
232243
THolder<TEvHive::TEvRequestHiveDomainStats> request = MakeHolder<TEvHive::TEvRequestHiveDomainStats>();
@@ -594,6 +605,19 @@ TViewerPipeClient::TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResu
594605
return response;
595606
}
596607

608+
TViewerPipeClient::TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>
609+
TViewerPipeClient::MakeRequestSchemeShardDescribe(TTabletId schemeShardId, const TString& path, const NKikimrSchemeOp::TDescribeOptions& options, ui64 cookie) {
610+
auto request = std::make_unique<NSchemeShard::TEvSchemeShard::TEvDescribeScheme>();
611+
request->Record.SetSchemeshardId(schemeShardId);
612+
request->Record.SetPath(path);
613+
request->Record.MutableOptions()->CopyFrom(options);
614+
auto response = MakeRequestToTablet<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>(schemeShardId, request.release(), cookie);
615+
if (response.Span) {
616+
response.Span.Attribute("path", path);
617+
}
618+
return response;
619+
}
620+
597621
void TViewerPipeClient::RequestTxProxyDescribe(const TString& path) {
598622
THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate());
599623
request->Record.MutableDescribePath()->SetPath(path);

‎ydb/core/viewer/json_pipe_req.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,17 @@ class TViewerPipeClient : public TActorBootstrapped<TViewerPipeClient> {
204204
return response;
205205
}
206206

207+
template<typename TResponse>
208+
TRequestResponse<TResponse> MakeRequestToTablet(TTabletId tabletId, IEventBase* ev, ui64 cookie = 0) {
209+
TActorId pipe = ConnectTabletPipe(tabletId);
210+
TRequestResponse<TResponse> response(Span.CreateChild(TComponentTracingLevels::THttp::Detailed, TypeName(*ev)));
211+
if (response.Span) {
212+
response.Span.Attribute("tablet_id", "#" + ::ToString(tabletId));
213+
}
214+
SendRequestToPipe(pipe, ev, cookie, response.Span.GetTraceId());
215+
return response;
216+
}
217+
207218
template<typename TRequest>
208219
TRequestResponse<typename NNodeWhiteboard::WhiteboardResponse<TRequest>::Type> MakeWhiteboardRequest(TNodeId nodeId, TRequest* ev, ui32 flags = IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession) {
209220
TActorId whiteboardServiceId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId);
@@ -241,6 +252,9 @@ class TViewerPipeClient : public TActorBootstrapped<TViewerPipeClient> {
241252
static bool IsSuccess(const std::unique_ptr<TEvStateStorage::TEvBoardInfo>& ev);
242253
static TString GetError(const std::unique_ptr<TEvStateStorage::TEvBoardInfo>& ev);
243254

255+
static bool IsSuccess(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev);
256+
static TString GetError(const std::unique_ptr<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>& ev);
257+
244258
TRequestResponse<TEvHive::TEvResponseHiveDomainStats> MakeRequestHiveDomainStats(TTabletId hiveId);
245259
TRequestResponse<TEvHive::TEvResponseHiveStorageStats> MakeRequestHiveStorageStats(TTabletId hiveId);
246260
TRequestResponse<TEvHive::TEvResponseHiveNodeStats> MakeRequestHiveNodeStats(TTabletId hiveId, TEvHive::TEvRequestHiveNodeStats* request);
@@ -270,6 +284,7 @@ class TViewerPipeClient : public TActorBootstrapped<TViewerPipeClient> {
270284
void RequestSchemeCacheNavigate(const TPathId& pathId);
271285
TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult> MakeRequestSchemeCacheNavigate(const TString& path, ui64 cookie = 0);
272286
TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult> MakeRequestSchemeCacheNavigate(TPathId pathId, ui64 cookie = 0);
287+
TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult> MakeRequestSchemeShardDescribe(TTabletId schemeShardId, const TString& path, const NKikimrSchemeOp::TDescribeOptions& options = {}, ui64 cookie = 0);
273288
TRequestResponse<TEvViewer::TEvViewerResponse> MakeRequestViewer(TNodeId nodeId, TEvViewer::TEvViewerRequest* request, ui32 flags = 0);
274289
void RequestTxProxyDescribe(const TString& path);
275290
void RequestStateStorageEndpointsLookup(const TString& path);

‎ydb/core/viewer/viewer_tenantinfo.h‎

Lines changed: 68 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ class TJsonTenantInfo : public TViewerPipeClient {
2222
std::optional<TRequestResponse<NConsole::TEvConsole::TEvListTenantsResponse>> ListTenantsResponse;
2323
std::unordered_map<TString, TRequestResponse<NConsole::TEvConsole::TEvGetTenantStatusResponse>> TenantStatusResponses;
2424
std::unordered_map<TString, TRequestResponse<TEvTxProxySchemeCache::TEvNavigateKeySetResult>> NavigateKeySetResult;
25+
std::unordered_map<TString, TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>> DescribeSchemeResult;
26+
std::unordered_map<TTabletId, std::vector<TString>> DescribesBySchemeShard;
2527
std::unordered_map<TTabletId, TRequestResponse<TEvHive::TEvResponseHiveDomainStats>> HiveDomainStats;
2628
std::unordered_map<TTabletId, TRequestResponse<TEvHive::TEvResponseHiveStorageStats>> HiveStorageStats;
2729
std::unordered_map<TNodeId, TRequestResponse<TEvWhiteboard::TEvSystemStateResponse>> SystemStateResponse;
@@ -52,6 +54,7 @@ class TJsonTenantInfo : public TViewerPipeClient {
5254
bool MetadataCache = true;
5355
THashMap<TString, std::vector<TNodeId>> TenantNodes;
5456
TTabletId RootHiveId = 0;
57+
TTabletId RootSchemeShardId = 0;
5558
TString RootId; // id of root domain (tenant)
5659
NKikimrViewer::TTenantInfo Result;
5760

@@ -97,6 +100,15 @@ class TJsonTenantInfo : public TViewerPipeClient {
97100
}
98101
}
99102

103+
TRequestResponse<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult> MakeRequestSchemeShardDescribe(TTabletId schemeShardId, const TString& path) {
104+
NKikimrSchemeOp::TDescribeOptions options;
105+
options.SetReturnPartitioningInfo(false);
106+
options.SetReturnPartitionConfig(false);
107+
options.SetReturnChildren(false);
108+
options.SetReturnRangeKey(false);
109+
return TBase::MakeRequestSchemeShardDescribe(schemeShardId, path, options);
110+
}
111+
100112
void Bootstrap() override {
101113
if (NeedToRedirect()) {
102114
return;
@@ -120,6 +132,7 @@ class TJsonTenantInfo : public TViewerPipeClient {
120132
TIntrusivePtr<TDomainsInfo> domains = AppData()->DomainsInfo;
121133
auto* domain = domains->GetDomain();
122134
DomainPath = "/" + domain->Name;
135+
RootSchemeShardId = domain->SchemeRoot;
123136
TPathId rootPathId(domain->SchemeRoot, 1);
124137
RootId = GetDomainId(rootPathId);
125138
RootHiveId = domains->GetHive();
@@ -146,6 +159,8 @@ class TJsonTenantInfo : public TViewerPipeClient {
146159
HiveDomainStats[RootHiveId] = MakeRequestHiveDomainStats(RootHiveId);
147160
if (Storage) {
148161
HiveStorageStats[RootHiveId] = MakeRequestHiveStorageStats(RootHiveId);
162+
DescribeSchemeResult[DomainPath] = MakeRequestSchemeShardDescribe(RootSchemeShardId, DomainPath);
163+
DescribesBySchemeShard[RootSchemeShardId].emplace_back(DomainPath);
149164
}
150165

151166
Become(&TThis::StateCollectingInfo, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup());
@@ -175,6 +190,7 @@ class TJsonTenantInfo : public TViewerPipeClient {
175190
hFunc(TEvTabletPipe::TEvClientConnected, Handle);
176191
hFunc(TEvStateStorage::TEvBoardInfo, Handle);
177192
hFunc(NHealthCheck::TEvSelfCheckResultProto, Handle);
193+
hFunc(TEvSchemeShard::TEvDescribeSchemeResult, Handle);
178194
cFunc(TEvents::TSystem::Wakeup, HandleTimeout);
179195
}
180196
}
@@ -202,6 +218,12 @@ class TJsonTenantInfo : public TViewerPipeClient {
202218
it->second.Error(error);
203219
}
204220
}
221+
auto itDescribes = DescribesBySchemeShard.find(ev->Get()->TabletId);
222+
if (itDescribes != DescribesBySchemeShard.end()) {
223+
for (const TString& path : itDescribes->second) {
224+
DescribeSchemeResult[path].Error(error);
225+
}
226+
}
205227
}
206228
TBase::Handle(ev); // all RequestDone() are handled by base handler
207229
}
@@ -397,14 +419,19 @@ class TJsonTenantInfo : public TViewerPipeClient {
397419
if (result.IsOk()) {
398420
auto domainInfo = result.Get()->Request->ResultSet.begin()->DomainInfo;
399421
TTabletId hiveId = domainInfo->Params.GetHive();
422+
TTabletId schemeShardId = domainInfo->Params.GetSchemeShard();
400423
if (hiveId) {
401424
if (HiveDomainStats.count(hiveId) == 0) {
402425
HiveDomainStats[hiveId] = MakeRequestHiveDomainStats(hiveId);
403426
}
404-
if (Storage) {
405-
if (HiveStorageStats.count(hiveId) == 0) {
406-
HiveStorageStats[hiveId] = MakeRequestHiveStorageStats(hiveId);
407-
}
427+
}
428+
if (Storage) {
429+
if (hiveId && HiveStorageStats.count(hiveId) == 0) {
430+
HiveStorageStats[hiveId] = MakeRequestHiveStorageStats(hiveId);
431+
}
432+
if (schemeShardId && DescribeSchemeResult.count(path) == 0) {
433+
DescribeSchemeResult[path] = MakeRequestSchemeShardDescribe(schemeShardId, path);
434+
DescribesBySchemeShard[schemeShardId].emplace_back(path);
408435
}
409436
}
410437
NKikimrViewer::TTenant& tenant = TenantBySubDomainKey[domainInfo->DomainKey];
@@ -424,6 +451,13 @@ class TJsonTenantInfo : public TViewerPipeClient {
424451
}
425452
}
426453

454+
void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) {
455+
TString path = ev->Get()->GetRecord().GetPath();
456+
if (DescribeSchemeResult[path].Set(std::move(ev))) {
457+
RequestDone();
458+
}
459+
}
460+
427461
void Handle(NNodeWhiteboard::TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) {
428462
ui32 nodeId = ev.Get()->Cookie;
429463
if (SystemStateResponse[nodeId].Set(std::move(ev))) {
@@ -745,7 +779,6 @@ class TJsonTenantInfo : public TViewerPipeClient {
745779
}
746780
}
747781

748-
THashMap<NKikimrViewer::TStorageUsage::EType, ui64> tablesStorageByType;
749782
THashMap<NKikimrViewer::TStorageUsage::EType, TStorageQuota> storageQuotasByType;
750783

751784
for (const auto& quota : tenant.GetDatabaseQuotas().storage_quotas()) {
@@ -755,42 +788,47 @@ class TJsonTenantInfo : public TViewerPipeClient {
755788
usage.HardQuota += quota.data_size_hard_quota();
756789
}
757790

758-
if (entry.DomainDescription) {
759-
for (const auto& poolUsage : entry.DomainDescription->Description.GetDiskSpaceUsage().GetStoragePoolsUsage()) {
791+
auto itDescribeScheme = DescribeSchemeResult.find(name);
792+
if (itDescribeScheme != DescribeSchemeResult.end() && itDescribeScheme->second.IsOk()) {
793+
const auto& record = itDescribeScheme->second.Get()->GetRecord();
794+
const auto& domainDescription(record.GetPathDescription().GetDomainDescription());
795+
THashMap<NKikimrViewer::TStorageUsage::EType, ui64> tablesStorageByType;
796+
797+
for (const auto& poolUsage : domainDescription.GetDiskSpaceUsage().GetStoragePoolsUsage()) {
760798
auto type = GetStorageType(poolUsage.GetPoolKind());
761799
tablesStorageByType[type] += poolUsage.GetTotalSize();
762800
}
763801

764-
if (tablesStorageByType.empty() && entry.DomainDescription->Description.HasDiskSpaceUsage()) {
765-
tablesStorageByType[GuessStorageType(entry.DomainDescription->Description)] =
766-
entry.DomainDescription->Description.GetDiskSpaceUsage().GetTables().GetTotalSize()
767-
+ entry.DomainDescription->Description.GetDiskSpaceUsage().GetTopics().GetDataSize();
802+
if (tablesStorageByType.empty() && domainDescription.HasDiskSpaceUsage()) {
803+
tablesStorageByType[GuessStorageType(domainDescription)] =
804+
domainDescription.GetDiskSpaceUsage().GetTables().GetTotalSize()
805+
+ domainDescription.GetDiskSpaceUsage().GetTopics().GetDataSize();
768806
}
769807

770808
if (storageQuotasByType.empty()) {
771-
auto& quotas = storageQuotasByType[GuessStorageType(entry.DomainDescription->Description)];
772-
quotas.HardQuota = entry.DomainDescription->Description.GetDatabaseQuotas().data_size_hard_quota();
773-
quotas.SoftQuota = entry.DomainDescription->Description.GetDatabaseQuotas().data_size_soft_quota();
809+
auto& quotas = storageQuotasByType[GuessStorageType(domainDescription)];
810+
quotas.HardQuota = domainDescription.GetDatabaseQuotas().data_size_hard_quota();
811+
quotas.SoftQuota = domainDescription.GetDatabaseQuotas().data_size_soft_quota();
774812
}
775-
}
776813

777-
for (const auto& [type, size] : tablesStorageByType) {
778-
auto it = storageQuotasByType.find(type);
779-
auto& tablesStorage = *tenant.AddTablesStorage();
780-
tablesStorage.SetType(type);
781-
tablesStorage.SetSize(size);
782-
if (it != storageQuotasByType.end()) {
783-
tablesStorage.SetLimit(it->second.SoftQuota);
784-
tablesStorage.SetSoftQuota(it->second.SoftQuota);
785-
tablesStorage.SetHardQuota(it->second.HardQuota);
814+
for (const auto& [type, size] : tablesStorageByType) {
815+
auto it = storageQuotasByType.find(type);
816+
auto& tablesStorage = *tenant.AddTablesStorage();
817+
tablesStorage.SetType(type);
818+
tablesStorage.SetSize(size);
819+
if (it != storageQuotasByType.end()) {
820+
tablesStorage.SetLimit(it->second.SoftQuota);
821+
tablesStorage.SetSoftQuota(it->second.SoftQuota);
822+
tablesStorage.SetHardQuota(it->second.HardQuota);
823+
}
786824
}
787-
}
788825

789-
for (const auto& [type, pr] : databaseStorageByType) {
790-
auto& databaseStorage = *tenant.AddDatabaseStorage();
791-
databaseStorage.SetType(type);
792-
databaseStorage.SetSize(pr.first);
793-
databaseStorage.SetLimit(pr.first + pr.second);
826+
for (const auto& [type, pr] : databaseStorageByType) {
827+
auto& databaseStorage = *tenant.AddDatabaseStorage();
828+
databaseStorage.SetType(type);
829+
databaseStorage.SetSize(pr.first);
830+
databaseStorage.SetLimit(pr.first + pr.second);
831+
}
794832
}
795833
}
796834

0 commit comments

Comments
 (0)