Skip to content

Commit 77f1571

Browse files
MetlinAlexanderMetlinAlexander
authored
Add throughput monitoring to request tracker (#4520)
Add throughput monitoring to request tracker(latency table) - Track operations and bytes per second for each request type (Read/Write/Zero/Describe) - Display ops/sec and bytes/sec metrics in Total latency tables <img width="1025" height="280" alt="image" src="https://github.com/user-attachments/assets/26785a1b-64d2-4bda-b21c-1695cb84988b" />
1 parent 5553713 commit 77f1571

File tree

4 files changed

+212
-3
lines changed

4 files changed

+212
-3
lines changed

‎cloud/blockstore/libs/storage/volume/model/requests_time_tracker.cpp‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,42 @@ bool TRequestsTimeTracker::TEqual::operator()(
153153

154154
////////////////////////////////////////////////////////////////////////////////
155155

156+
void TRequestsTimeTracker::TThroughputTracker::AddOperation(ui64 blockCount)
157+
{
158+
TotalBlockCount += blockCount;
159+
TotalOps++;
160+
}
161+
162+
std::pair<ui64, ui64>
163+
TRequestsTimeTracker::TThroughputTracker::GetRatesAndReset(
164+
ui64 currentTime,
165+
ui32 blockSize)
166+
{
167+
if (LastResetTime == 0) {
168+
LastResetTime = currentTime;
169+
return {0, 0};
170+
}
171+
172+
const ui64 elapsedCycles = currentTime - LastResetTime;
173+
const ui64 elapsedUs = CyclesToDurationSafe(elapsedCycles).MicroSeconds();
174+
175+
if (elapsedUs == 0) {
176+
return {0, 0};
177+
}
178+
179+
const ui64 totalBytes = TotalBlockCount * static_cast<ui64>(blockSize);
180+
const ui64 bytesPerSecond = (totalBytes * 1000000ULL) / elapsedUs;
181+
const ui64 opsPerSecond = (TotalOps * 1000000ULL) / elapsedUs;
182+
183+
TotalBlockCount = 0;
184+
TotalOps = 0;
185+
LastResetTime = currentTime;
186+
187+
return {bytesPerSecond, opsPerSecond};
188+
}
189+
190+
////////////////////////////////////////////////////////////////////////////////
191+
156192
TRequestsTimeTracker::TRequestsTimeTracker(const ui64 constructionTime)
157193
: ConstructionTime(constructionTime)
158194
{
@@ -168,6 +204,10 @@ TRequestsTimeTracker::TRequestsTimeTracker(const ui64 constructionTime)
168204
Histograms[key];
169205
}
170206
}
207+
208+
for (auto& tracker: ThroughputCounters) {
209+
tracker.LastResetTime = constructionTime;
210+
}
171211
}
172212

173213
void TRequestsTimeTracker::OnRequestStarted(
@@ -250,6 +290,12 @@ TRequestsTimeTracker::OnRequestFinished(
250290
Histograms[key].Increment(duration.MicroSeconds());
251291
Histograms[key].BlockCount += request.BlockRange.Size();
252292

293+
const auto requestTypeIndex = static_cast<size_t>(request.RequestType);
294+
if (requestTypeIndex < ThroughputCounters.size()) {
295+
ThroughputCounters.at(requestTypeIndex)
296+
.AddOperation(request.BlockRange.Size());
297+
}
298+
253299
return StatFirstSuccess(request, success, finishTime);
254300
}
255301

@@ -288,7 +334,7 @@ NJson::TJsonValue TRequestsTimeTracker::BuildPercentilesJson() const
288334
return result;
289335
}
290336

291-
TString TRequestsTimeTracker::GetStatJson(ui64 nowCycles, ui32 blockSize) const
337+
TString TRequestsTimeTracker::GetStatJson(ui64 nowCycles, ui32 blockSize)
292338
{
293339
NJson::TJsonValue allStat(NJson::EJsonValueType::JSON_MAP);
294340

@@ -348,6 +394,27 @@ TString TRequestsTimeTracker::GetStatJson(ui64 nowCycles, ui32 blockSize) const
348394
}
349395
}
350396

397+
const char* typeNames[] = {"R", "W", "Z", "D"};
398+
399+
for (size_t requestType = 0; requestType < ThroughputCounters.size();
400+
++requestType)
401+
{
402+
auto [bytesPerSec, opsPerSec] =
403+
ThroughputCounters.at(requestType)
404+
.GetRatesAndReset(nowCycles, blockSize);
405+
406+
const TString formattedThroughput =
407+
bytesPerSec > 0 ? (FormatByteSize(bytesPerSec) + "/s") : "0 B/s";
408+
409+
const TString htmlKeyBytes =
410+
TString(typeNames[requestType]) + "_Total_BytesPerSec";
411+
const TString htmlKeyOps =
412+
TString(typeNames[requestType]) + "_Total_OpsPerSec";
413+
414+
allStat[htmlKeyBytes] = formattedThroughput;
415+
allStat[htmlKeyOps] = ToString(opsPerSec);
416+
}
417+
351418
NJson::TJsonValue json;
352419
json["stat"] = std::move(allStat);
353420
json["percentiles"] = BuildPercentilesJson();

‎cloud/blockstore/libs/storage/volume/model/requests_time_tracker.h‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,20 @@ class TRequestsTimeTracker
113113
bool success,
114114
ui64 finishTime);
115115

116+
struct TThroughputTracker
117+
{
118+
ui64 LastResetTime = 0;
119+
ui64 TotalBlockCount = 0;
120+
ui64 TotalOps = 0;
121+
122+
void AddOperation(ui64 blockCount);
123+
std::pair<ui64, ui64> GetRatesAndReset(
124+
ui64 currentTime,
125+
ui32 blockSize);
126+
};
127+
128+
std::array<TThroughputTracker, RequestTypeCount> ThroughputCounters;
129+
116130
public:
117131
explicit TRequestsTimeTracker(const ui64 constructionTime);
118132

@@ -131,7 +145,7 @@ class TRequestsTimeTracker
131145
[[nodiscard]] std::optional<TFirstSuccessStat>
132146
OnRequestFinished(ui64 requestId, bool success, ui64 finishTime);
133147

134-
[[nodiscard]] TString GetStatJson(ui64 nowCycles, ui32 blockSize) const;
148+
[[nodiscard]] TString GetStatJson(ui64 nowCycles, ui32 blockSize);
135149

136150
void ResetStats();
137151

‎cloud/blockstore/libs/storage/volume/model/requests_time_tracker_ut.cpp‎

Lines changed: 92 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ size_t DumpValues(const NJson::TJsonValue::TMapType& map)
2121

2222
size_t nonEmptyCount = 0;
2323
for (const auto& [key, val]: ordered) {
24-
if (val != "" && val != "0" && val != "0 B") {
24+
if (val != "" && val != "0" && val != "0 B" &&
25+
!key.EndsWith("OpsPerSec") && !key.EndsWith("BytesPerSec"))
26+
{
2527
Cout << key << "=" << val << Endl;
2628
++nonEmptyCount;
2729
}
@@ -340,6 +342,95 @@ Y_UNIT_TEST_SUITE(TRequestsTimeTrackerTest)
340342
"1",
341343
valueAfter["stat"]["W_1_inflight_Count"].GetString());
342344
}
345+
346+
Y_UNIT_TEST(ShouldCalculateThroughputStatistics)
347+
{
348+
const ui64 baseTime = 1000000000000ULL;
349+
TRequestsTimeTracker requestsTimeTracker(baseTime);
350+
351+
requestsTimeTracker.OnRequestStarted(
352+
TRequestsTimeTracker::ERequestType::Read,
353+
1,
354+
TBlockRange64::WithLength(0, 4),
355+
baseTime + 100000);
356+
357+
requestsTimeTracker.OnRequestStarted(
358+
TRequestsTimeTracker::ERequestType::Read,
359+
2,
360+
TBlockRange64::WithLength(0, 8),
361+
baseTime + 200000);
362+
363+
requestsTimeTracker.OnRequestStarted(
364+
TRequestsTimeTracker::ERequestType::Write,
365+
3,
366+
TBlockRange64::WithLength(0, 16),
367+
baseTime + 300000);
368+
369+
Y_UNUSED(requestsTimeTracker.GetStatJson(baseTime + 350000, 4096));
370+
371+
auto readStat1 =
372+
requestsTimeTracker.OnRequestFinished(1, true, baseTime + 400000);
373+
auto readStat2 =
374+
requestsTimeTracker.OnRequestFinished(2, true, baseTime + 500000);
375+
auto writeStat =
376+
requestsTimeTracker.OnRequestFinished(3, true, baseTime + 600000);
377+
378+
UNIT_ASSERT(readStat1.has_value());
379+
UNIT_ASSERT(!readStat2.has_value());
380+
UNIT_ASSERT(writeStat.has_value());
381+
382+
auto json =
383+
requestsTimeTracker.GetStatJson(baseTime + 1000100000, 4096);
384+
385+
NJson::TJsonValue value;
386+
NJson::ReadJsonTree(json, &value, true);
387+
388+
auto getStat = [&](const TString& key) -> TString
389+
{
390+
if (!value["stat"].Has(key)) {
391+
return "KEY_MISSING";
392+
}
393+
return value["stat"][key].GetString();
394+
};
395+
396+
auto readOps = getStat("R_Total_OpsPerSec");
397+
auto writeOps = getStat("W_Total_OpsPerSec");
398+
399+
UNIT_ASSERT(readOps != "KEY_MISSING");
400+
UNIT_ASSERT(writeOps != "KEY_MISSING");
401+
UNIT_ASSERT(readOps != "0");
402+
UNIT_ASSERT(writeOps != "0");
403+
404+
auto readBytes = getStat("R_Total_BytesPerSec");
405+
auto writeBytes = getStat("W_Total_BytesPerSec");
406+
UNIT_ASSERT(readBytes != "0 B/s");
407+
UNIT_ASSERT(writeBytes != "0 B/s");
408+
UNIT_ASSERT(readBytes.Contains("/s"));
409+
UNIT_ASSERT(writeBytes.Contains("/s"));
410+
411+
UNIT_ASSERT_VALUES_EQUAL("0", getStat("Z_Total_OpsPerSec"));
412+
UNIT_ASSERT_VALUES_EQUAL("0 B/s", getStat("Z_Total_BytesPerSec"));
413+
UNIT_ASSERT_VALUES_EQUAL("0", getStat("D_Total_OpsPerSec"));
414+
UNIT_ASSERT_VALUES_EQUAL("0 B/s", getStat("D_Total_BytesPerSec"));
415+
416+
auto json2 =
417+
requestsTimeTracker.GetStatJson(baseTime + 2000200000, 4096);
418+
NJson::TJsonValue value2;
419+
NJson::ReadJsonTree(json2, &value2, true);
420+
421+
auto getStat2 = [&](const TString& key) -> TString
422+
{
423+
if (!value2["stat"].Has(key)) {
424+
return "KEY_MISSING";
425+
}
426+
return value2["stat"][key].GetString();
427+
};
428+
429+
UNIT_ASSERT_VALUES_EQUAL("0", getStat2("R_Total_OpsPerSec"));
430+
UNIT_ASSERT_VALUES_EQUAL("0 B/s", getStat2("R_Total_BytesPerSec"));
431+
UNIT_ASSERT_VALUES_EQUAL("0", getStat2("W_Total_OpsPerSec"));
432+
UNIT_ASSERT_VALUES_EQUAL("0 B/s", getStat2("W_Total_BytesPerSec"));
433+
}
343434
}
344435

345436
} // namespace NCloud::NBlockStore::NStorage

‎cloud/blockstore/libs/storage/volume/volume_actor_monitoring.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,43 @@ void RenderLatencyTable(IOutputStream& out, const TString& parentId)
609609
}
610610
}
611611
}
612+
if (parentId.Contains("Total")) {
613+
TABLER () {
614+
TABLED () {
615+
RenderTextWithTooltip(
616+
out,
617+
"Ops/Sec",
618+
"Operations per second");
619+
}
620+
TABLED_ATTRS ({{"id", parentId + "_OpsPerSec"}}) {
621+
out << "0";
622+
}
623+
TABLED () {
624+
out << "-";
625+
}
626+
TABLED () {
627+
out << "-";
628+
}
629+
}
630+
631+
TABLER () {
632+
TABLED () {
633+
RenderTextWithTooltip(
634+
out,
635+
"Data/Sec",
636+
"Data throughput per second");
637+
}
638+
TABLED_ATTRS ({{"id", parentId + "_BytesPerSec"}}) {
639+
out << "0 B/s";
640+
}
641+
TABLED () {
642+
out << "-";
643+
}
644+
TABLED () {
645+
out << "-";
646+
}
647+
}
648+
}
612649
}
613650
}
614651
}

0 commit comments

Comments
 (0)