Skip to content

Commit 52f0bb1

Browse files
ydbotydbot
authored
stable-25-3-1: Add PDisk metadata tool (#26321)
2 parents 2bf0f3d + 1ce90f3 commit 52f0bb1

File tree

10 files changed

+803
-9
lines changed

10 files changed

+803
-9
lines changed

‎ydb/core/blobstorage/pdisk/metadata/blobstorage_pdisk_metadata.cpp‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
#include "blobstorage_pdisk_metadata.h"
2+
#include <ydb/core/blobstorage/pdisk/blobstorage_pdisk.h>
3+
#include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_actorsystem_creator.h>
4+
#include <ydb/library/actors/core/actor_bootstrapped.h>
5+
#include <library/cpp/openssl/crypto/sha.h>
6+
#include <future>
7+
8+
namespace NKikimr {
9+
namespace NPDisk {
10+
11+
namespace {
12+
constexpr ui32 kNodeIdForMetadata = 1;
13+
constexpr ui32 kPDiskIdForMetadata = 1;
14+
constexpr ui32 kMetaTimeoutSeconds = 10;
15+
16+
static NPDisk::TMainKey GetEffectiveMainKey(const NPDisk::TMainKey& mainKey) {
17+
NPDisk::TMainKey result = mainKey;
18+
if (!result.IsInitialized || result.Keys.empty()) {
19+
result.Initialize();
20+
}
21+
return result;
22+
}
23+
24+
static TIntrusivePtr<TPDiskConfig> MakeMetadataPDiskConfig(const TString& path, ui32 pdiskId, bool readOnly, TStringStream& details) {
25+
std::optional<NPDisk::TDriveData> driveData = NPDisk::GetDriveData(path, &details);
26+
const NPDisk::EDeviceType deviceType = driveData ? driveData->DeviceType : NPDisk::DEVICE_TYPE_ROT;
27+
const ui64 category = static_cast<ui64>(TPDiskCategory(deviceType, 0).GetRaw());
28+
29+
auto cfg = MakeIntrusive<TPDiskConfig>(path, static_cast<ui64>(0), static_cast<ui32>(pdiskId), category);
30+
cfg->ReadOnly = readOnly;
31+
cfg->MetadataOnly = true;
32+
return cfg;
33+
}
34+
35+
static NActors::TActorId RegisterPDiskActor(NActors::TActorSystem* sys,
36+
const TIntrusivePtr<TPDiskConfig>& cfg,
37+
const NPDisk::TMainKey& mainKey,
38+
const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) {
39+
IActor* pdiskActorImpl = CreatePDisk(cfg, mainKey, counters);
40+
const NActors::TActorId pdiskActor = sys->Register(pdiskActorImpl);
41+
const NActors::TActorId pdiskActorId = MakeBlobStoragePDiskID(/*nodeId*/kNodeIdForMetadata, /*pdiskId*/kPDiskIdForMetadata);
42+
sys->RegisterLocalService(pdiskActorId, pdiskActor);
43+
return pdiskActorId;
44+
}
45+
46+
template <typename TFut>
47+
static inline void WaitOrThrow(TFut& fut, int timeoutSeconds, const TString& details, const char* action) {
48+
if (fut.wait_for(std::chrono::seconds(timeoutSeconds)) != std::future_status::ready) {
49+
ythrow yexception()
50+
<< "PDisk metadata " << action << " timeout after " << timeoutSeconds << "s\n"
51+
<< "Details: " << details;
52+
}
53+
}
54+
55+
} // namespace
56+
57+
NKikimrBlobStorage::TPDiskMetadataRecord ReadPDiskMetadata(const TString& path, const NPDisk::TMainKey& mainKey) {
58+
TActorSystemCreator creator;
59+
auto* sys = creator.GetActorSystem();
60+
auto counters = MakeIntrusive<::NMonitoring::TDynamicCounters>();
61+
62+
TStringStream details;
63+
64+
auto pdiskCfg = MakeMetadataPDiskConfig(path, kPDiskIdForMetadata, /*readOnly*/true, details);
65+
66+
const NPDisk::TMainKey effectiveMainKey = GetEffectiveMainKey(mainKey);
67+
68+
const NActors::TActorId pdiskActorId = RegisterPDiskActor(sys, pdiskCfg, effectiveMainKey, counters);
69+
70+
NKikimrBlobStorage::TPDiskMetadataRecord out;
71+
72+
class TMetadataReader : public NActors::TActorBootstrapped<TMetadataReader> {
73+
const NActors::TActorId PDiskActorId;
74+
NKikimrBlobStorage::TPDiskMetadataRecord& Out;
75+
std::promise<bool>& Done;
76+
TString& Error;
77+
78+
static const char* OutcomeToStr(NPDisk::EPDiskMetadataOutcome oc) {
79+
switch (oc) {
80+
case NPDisk::EPDiskMetadataOutcome::OK: return "OK";
81+
case NPDisk::EPDiskMetadataOutcome::NO_METADATA: return "NO_METADATA";
82+
case NPDisk::EPDiskMetadataOutcome::ERROR: return "ERROR";
83+
}
84+
return "UNKNOWN";
85+
}
86+
public:
87+
TMetadataReader(NActors::TActorId pdiskActorId,
88+
NKikimrBlobStorage::TPDiskMetadataRecord& out,
89+
std::promise<bool>& done,
90+
TString& error)
91+
: PDiskActorId(pdiskActorId)
92+
, Out(out)
93+
, Done(done)
94+
, Error(error)
95+
{}
96+
97+
void Bootstrap() {
98+
Send(PDiskActorId, new NPDisk::TEvReadMetadata());
99+
Become(&TThis::StateFunc, TDuration::Seconds(kMetaTimeoutSeconds), new TEvents::TEvWakeup);
100+
}
101+
102+
void Handle(NPDisk::TEvReadMetadataResult::TPtr ev) {
103+
auto* msg = ev->Get();
104+
bool ok = false;
105+
switch (msg->Outcome) {
106+
case NPDisk::EPDiskMetadataOutcome::OK: {
107+
TRope rope(std::move(msg->Metadata));
108+
TRopeStream stream(rope.begin(), rope.size());
109+
ok = Out.ParseFromZeroCopyStream(&stream);
110+
if (!ok) {
111+
Error = "PARSE_FAILED";
112+
}
113+
break;
114+
}
115+
case NPDisk::EPDiskMetadataOutcome::NO_METADATA: {
116+
Error = OutcomeToStr(msg->Outcome);
117+
ok = false;
118+
break;
119+
}
120+
case NPDisk::EPDiskMetadataOutcome::ERROR: {
121+
Error = OutcomeToStr(msg->Outcome);
122+
ok = false;
123+
break;
124+
}
125+
}
126+
Done.set_value(ok);
127+
PassAway();
128+
}
129+
130+
void HandleWakeup() {
131+
Error = "TIMEOUT";
132+
Done.set_value(false);
133+
PassAway();
134+
}
135+
136+
STRICT_STFUNC(StateFunc,
137+
hFunc(NPDisk::TEvReadMetadataResult, Handle);
138+
cFunc(TEvents::TSystem::Wakeup, HandleWakeup);
139+
)
140+
};
141+
142+
std::promise<bool> done;
143+
auto fut = done.get_future();
144+
TString error;
145+
sys->Register(new TMetadataReader(pdiskActorId, out, done, error));
146+
147+
WaitOrThrow(fut, 10, details.Str(), "read");
148+
bool ok = fut.get();
149+
if (!ok) {
150+
ythrow yexception()
151+
<< "PDisk metadata read failed: " << (!error.empty() ? error : TString("ERROR")) << "\n"
152+
<< "Details: " << details.Str();
153+
}
154+
return out;
155+
}
156+
157+
static void UpdateStorageConfigFingerprint(NKikimrBlobStorage::TStorageConfig* config) {
158+
if (!config) {
159+
return;
160+
}
161+
config->ClearFingerprint();
162+
TString s;
163+
const bool success = config->SerializeToString(&s);
164+
Y_ABORT_UNLESS(success);
165+
auto digest = NOpenSsl::NSha1::Calc(s.data(), s.size());
166+
config->SetFingerprint(digest.data(), digest.size());
167+
}
168+
169+
void WritePDiskMetadata(const TString& path, const NKikimrBlobStorage::TPDiskMetadataRecord& record, const NPDisk::TMainKey& mainKey) {
170+
NKikimrBlobStorage::TPDiskMetadataRecord adjustedRecord(record);
171+
172+
const NPDisk::TMainKey effectiveMainKey = GetEffectiveMainKey(mainKey);
173+
174+
NKikimrBlobStorage::TPDiskMetadataRecord previous;
175+
bool havePrevious = false;
176+
try {
177+
previous = ReadPDiskMetadata(path, effectiveMainKey);
178+
havePrevious = true;
179+
} catch (...) {
180+
havePrevious = false;
181+
}
182+
183+
if (adjustedRecord.HasCommittedStorageConfig()) {
184+
auto* cfg = adjustedRecord.MutableCommittedStorageConfig();
185+
if (!cfg->HasPrevConfig() && havePrevious) {
186+
if (previous.HasCommittedStorageConfig()) {
187+
cfg->MutablePrevConfig()->CopyFrom(previous.GetCommittedStorageConfig());
188+
}
189+
}
190+
UpdateStorageConfigFingerprint(cfg);
191+
}
192+
193+
TActorSystemCreator creator;
194+
auto* sys = creator.GetActorSystem();
195+
auto counters = MakeIntrusive<::NMonitoring::TDynamicCounters>();
196+
197+
TStringStream details;
198+
199+
auto pdiskCfg = MakeMetadataPDiskConfig(path, kPDiskIdForMetadata, /*readOnly*/false, details);
200+
201+
const NActors::TActorId pdiskActorId = RegisterPDiskActor(sys, pdiskCfg, effectiveMainKey, counters);
202+
203+
class TWriter : public NActors::TActorBootstrapped<TWriter> {
204+
const NActors::TActorId PDiskActorId;
205+
const NKikimrBlobStorage::TPDiskMetadataRecord& Record;
206+
std::promise<bool>& Done;
207+
public:
208+
TWriter(NActors::TActorId pdiskActorId,
209+
const NKikimrBlobStorage::TPDiskMetadataRecord& record,
210+
std::promise<bool>& done)
211+
: PDiskActorId(pdiskActorId)
212+
, Record(record)
213+
, Done(done)
214+
{}
215+
216+
void Bootstrap() {
217+
TString data;
218+
if (!Record.SerializeToString(&data)) {
219+
Done.set_value(false);
220+
PassAway();
221+
return;
222+
}
223+
Send(PDiskActorId, new NPDisk::TEvWriteMetadata(TRcBuf(std::move(data))));
224+
Become(&TThis::StateFunc, TDuration::Seconds(kMetaTimeoutSeconds), new TEvents::TEvWakeup);
225+
}
226+
227+
void Handle(NPDisk::TEvWriteMetadataResult::TPtr ev) {
228+
bool ok = (ev->Get()->Outcome == NPDisk::EPDiskMetadataOutcome::OK);
229+
Done.set_value(ok);
230+
PassAway();
231+
}
232+
233+
void HandleWakeup() {
234+
Done.set_value(false);
235+
PassAway();
236+
}
237+
238+
STRICT_STFUNC(StateFunc,
239+
hFunc(NPDisk::TEvWriteMetadataResult, Handle);
240+
cFunc(TEvents::TSystem::Wakeup, HandleWakeup);
241+
)
242+
};
243+
244+
std::promise<bool> done;
245+
auto fut = done.get_future();
246+
sys->Register(new TWriter(pdiskActorId, adjustedRecord, done));
247+
248+
WaitOrThrow(fut, 10, details.Str(), "write");
249+
if (!fut.get()) {
250+
ythrow yexception()
251+
<< "PDisk metadata write failed\n"
252+
<< "Details: " << details.Str();
253+
}
254+
}
255+
256+
} // NPDisk
257+
} // NKikimr

‎ydb/core/blobstorage/pdisk/metadata/blobstorage_pdisk_metadata.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#pragma once
2+
3+
#include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_defs.h>
4+
#include <ydb/core/protos/blobstorage_distributed_config.pb.h>
5+
6+
namespace NKikimr {
7+
namespace NPDisk {
8+
9+
// reads metadata from PDisk (if available)
10+
NKikimrBlobStorage::TPDiskMetadataRecord ReadPDiskMetadata(const TString& path, const TMainKey& mainKey);
11+
12+
// writes metadata record to PDisk (throws on error)
13+
void WritePDiskMetadata(const TString& path, const NKikimrBlobStorage::TPDiskMetadataRecord& record, const TMainKey& mainKey);
14+
15+
} // NPDisk
16+
} // NKikimr

‎ydb/core/blobstorage/pdisk/metadata/ya.make‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
LIBRARY()
2+
3+
SRCS(
4+
blobstorage_pdisk_metadata.h
5+
blobstorage_pdisk_metadata.cpp
6+
)
7+
8+
PEERDIR(
9+
ydb/core/base
10+
ydb/core/blobstorage/base
11+
ydb/core/blobstorage/crypto
12+
ydb/core/blobstorage/nodewarden
13+
ydb/core/blobstorage/pdisk
14+
ydb/library/actors/core
15+
ydb/library/keys
16+
)
17+
18+
END()

‎ydb/core/blobstorage/pdisk/ya.make‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ SRCS(
7474
END()
7575

7676
RECURSE(
77+
metadata
7778
mock
7879
)
7980

0 commit comments

Comments
 (0)