Skip to content

Commit c826e15

Browse files
ifed01ifed01
committed
common/perf_counters: track maximum increment value for LONGRUNAVG perf
counters This e.g. allows to track maximum observed latencies along with average values which might be helpful for bottleneck detection. Signed-off-by: Igor Fedotov <[email protected]>
1 parent 07ab7ec commit c826e15

File tree

3 files changed

+122
-16
lines changed

3 files changed

+122
-16
lines changed

‎src/common/perf_counters.cc‎

Lines changed: 96 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,31 @@ void PerfCounters::inc(int idx, uint64_t amt)
183183
}
184184
}
185185

186+
void PerfCounters::inc_with_max(int idx, uint64_t amt)
187+
{
188+
#ifndef WITH_CRIMSON
189+
if (!m_cct->_conf->perf)
190+
return;
191+
#endif
192+
193+
ceph_assert(idx > m_lower_bound);
194+
ceph_assert(idx < m_upper_bound);
195+
perf_counter_data_any_d& data(m_data[idx - m_lower_bound - 1]);
196+
if (!(data.type & PERFCOUNTER_U64))
197+
return;
198+
if (data.type & PERFCOUNTER_LONGRUNAVG) {
199+
data.avgcount++;
200+
data.u64 += amt;
201+
uint64_t m;
202+
do {
203+
m = data.max_u64_inc.load();
204+
} while(amt > m && !data.max_u64_inc.compare_exchange_weak(m, amt));
205+
data.avgcount2++;
206+
} else {
207+
data.u64 += amt;
208+
}
209+
}
210+
186211
void PerfCounters::dec(int idx, uint64_t amt)
187212
{
188213
#ifndef WITH_CRIMSON
@@ -259,6 +284,32 @@ void PerfCounters::tinc(int idx, utime_t amt)
259284
}
260285
}
261286

287+
void PerfCounters::tinc_with_max(int idx, utime_t amt)
288+
{
289+
#ifndef WITH_CRIMSON
290+
if (!m_cct->_conf->perf)
291+
return;
292+
#endif
293+
294+
ceph_assert(idx > m_lower_bound);
295+
ceph_assert(idx < m_upper_bound);
296+
perf_counter_data_any_d& data(m_data[idx - m_lower_bound - 1]);
297+
if (!(data.type & PERFCOUNTER_TIME))
298+
return;
299+
if (data.type & PERFCOUNTER_LONGRUNAVG) {
300+
uint64_t new_m = amt.to_nsec();
301+
data.avgcount++;
302+
data.u64 += new_m;
303+
uint64_t m;
304+
do {
305+
m = data.max_u64_inc.load();
306+
} while(new_m > m && !data.max_u64_inc.compare_exchange_weak(m, new_m));
307+
data.avgcount2++;
308+
} else {
309+
data.u64 += amt.to_nsec();
310+
}
311+
}
312+
262313
void PerfCounters::tinc(int idx, ceph::timespan amt)
263314
{
264315
#ifndef WITH_CRIMSON
@@ -280,6 +331,32 @@ void PerfCounters::tinc(int idx, ceph::timespan amt)
280331
}
281332
}
282333

334+
void PerfCounters::tinc_with_max(int idx, ceph::timespan amt)
335+
{
336+
#ifndef WITH_CRIMSON
337+
if (!m_cct->_conf->perf)
338+
return;
339+
#endif
340+
341+
ceph_assert(idx > m_lower_bound);
342+
ceph_assert(idx < m_upper_bound);
343+
perf_counter_data_any_d& data(m_data[idx - m_lower_bound - 1]);
344+
if (!(data.type & PERFCOUNTER_TIME))
345+
return;
346+
if (data.type & PERFCOUNTER_LONGRUNAVG) {
347+
uint64_t new_m = amt.count();
348+
data.avgcount++;
349+
data.u64 += new_m;
350+
uint64_t m;
351+
do {
352+
m = data.max_u64_inc.load();
353+
} while(new_m > m && !data.max_u64_inc.compare_exchange_weak(m, new_m));
354+
data.avgcount2++;
355+
} else {
356+
data.u64 += amt.count();
357+
}
358+
}
359+
283360
void PerfCounters::tset(int idx, utime_t amt)
284361
{
285362
#ifndef WITH_CRIMSON
@@ -361,8 +438,7 @@ pair<uint64_t, uint64_t> PerfCounters::get_tavg_ns(int idx) const
361438
return make_pair(0, 0);
362439
if (!(data.type & PERFCOUNTER_LONGRUNAVG))
363440
return make_pair(0, 0);
364-
pair<uint64_t,uint64_t> a = data.read_avg();
365-
return make_pair(a.second, a.first);
441+
return data.read_avg();
366442
}
367443

368444
void PerfCounters::reset()
@@ -479,17 +555,27 @@ void PerfCounters::dump_formatted_generic(Formatter *f, bool schema,
479555
} else {
480556
if (d->type & PERFCOUNTER_LONGRUNAVG) {
481557
Formatter::ObjectSection longrunavg_section{*f, d->name};
482-
pair<uint64_t,uint64_t> a = d->read_avg();
558+
std::tuple<uint64_t,uint64_t,uint64_t> a = d->read_avg_ex();
483559
if (d->type & PERFCOUNTER_U64) {
484-
f->dump_unsigned("avgcount", a.second);
485-
f->dump_unsigned("sum", a.first);
560+
f->dump_unsigned("sum", std::get<0>(a));
561+
f->dump_unsigned("avgcount", std::get<1>(a));
562+
uint64_t max = std::get<2>(a);
563+
if (max != 0) {
564+
f->dump_unsigned("max_inc", std::get<2>(a));
565+
}
486566
} else if (d->type & PERFCOUNTER_TIME) {
487-
f->dump_unsigned("avgcount", a.second);
567+
uint64_t sum_ns = std::get<0>(a);
568+
uint64_t count = std::get<1>(a);
569+
f->dump_unsigned("avgcount", count);
488570
f->dump_format_unquoted("sum", "%" PRId64 ".%09" PRId64,
489-
a.first / 1000000000ull,
490-
a.first % 1000000000ull);
491-
uint64_t count = a.second;
492-
uint64_t sum_ns = a.first;
571+
sum_ns / 1000000000ull,
572+
sum_ns % 1000000000ull);
573+
uint64_t max_ns = std::get<2>(a);
574+
if (max_ns != 0) {
575+
f->dump_format_unquoted("max_inc", "%" PRId64 ".%09" PRId64,
576+
max_ns / 1000000000ull,
577+
max_ns % 1000000000ull);
578+
}
493579
if (count) {
494580
uint64_t avg_ns = sum_ns / count;
495581
f->dump_format_unquoted("avgtime", "%" PRId64 ".%09" PRId64,

‎src/common/perf_counters.h‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <set>
2323
#include <string>
2424
#include <vector>
25+
#include <tuple>
2526
#include <memory>
2627
#include <atomic>
2728
#include <cstdint>
@@ -178,12 +179,10 @@ class PerfCounters
178179
description(other.description),
179180
nick(other.nick),
180181
type(other.type),
181-
unit(other.unit),
182-
u64(other.u64.load()) {
183-
auto a = other.read_avg();
184-
u64 = a.first;
185-
avgcount = a.second;
186-
avgcount2 = a.second;
182+
unit(other.unit) {
183+
std::tie(u64, avgcount, max_u64_inc) = other.read_avg_ex();
184+
avgcount2 = avgcount.load();
185+
187186
if (other.histogram) {
188187
histogram.reset(new PerfHistogram<>(*other.histogram));
189188
}
@@ -196,6 +195,7 @@ class PerfCounters
196195
enum perfcounter_type_d type;
197196
enum unit_t unit;
198197
std::atomic<uint64_t> u64 = { 0 };
198+
std::atomic<uint64_t> max_u64_inc = { 0 };
199199
std::atomic<uint64_t> avgcount = { 0 };
200200
std::atomic<uint64_t> avgcount2 = { 0 };
201201
std::unique_ptr<PerfHistogram<>> histogram;
@@ -204,6 +204,7 @@ class PerfCounters
204204
{
205205
if (type != PERFCOUNTER_U64) {
206206
u64 = 0;
207+
max_u64_inc = 0;
207208
avgcount = 0;
208209
avgcount2 = 0;
209210
}
@@ -223,6 +224,15 @@ class PerfCounters
223224
} while (avgcount != count);
224225
return { sum, count };
225226
}
227+
std::tuple<uint64_t,uint64_t, uint64_t> read_avg_ex() const {
228+
uint64_t _sum, _count, _max;
229+
do {
230+
_count = avgcount2;
231+
_sum = u64;
232+
_max = max_u64_inc;
233+
} while (avgcount != _count);
234+
return { _sum, _count, _max };
235+
}
226236
};
227237

228238
template <typename T>
@@ -244,6 +254,7 @@ class PerfCounters
244254
~PerfCounters();
245255

246256
void inc(int idx, uint64_t v = 1);
257+
void inc_with_max(int idx, uint64_t v = 1);
247258
void dec(int idx, uint64_t v = 1);
248259
void set(int idx, uint64_t v);
249260
uint64_t get(int idx) const;
@@ -252,6 +263,8 @@ class PerfCounters
252263
void tset(int idx, ceph::timespan v);
253264
void tinc(int idx, utime_t v);
254265
void tinc(int idx, ceph::timespan v);
266+
void tinc_with_max(int idx, utime_t v);
267+
void tinc_with_max(int idx, ceph::timespan v);
255268
utime_t tget(int idx) const;
256269

257270
void hinc(int idx, int64_t x, int64_t y);

‎src/test/perf_counters.cc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,13 @@ TEST(PerfCounters, SinglePerfCounters) {
123123
ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1,\"element2\":0.500000000,"
124124
"\"element3\":{\"avgcount\":3,\"sum\":120.000000000,\"avgtime\":40.000000000}}}"), msg);
125125

126+
fake_pf->reset();
127+
fake_pf->tinc_with_max(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(100, 0));
128+
fake_pf->tinc_with_max(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(50, 0));
129+
ASSERT_EQ("", client.do_request("{ \"prefix\": \"perf dump\", \"format\": \"json\" }", &msg));
130+
ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1,"
131+
"\"element2\":0.000000000,\"element3\":{\"avgcount\":2,\"sum\":150.000000000,\"max_inc\":100.000000000,\"avgtime\":75.000000000}}}"), msg);
132+
126133
fake_pf->reset();
127134
msg.clear();
128135
ASSERT_EQ("", client.do_request("{ \"prefix\": \"perf dump\", \"format\": \"json\" }", &msg));

0 commit comments

Comments
 (0)