Skip to content

Commit b3777d0

Browse files
jameseh96jameseh96
committed
MB-39046: Introduce units for statistics
While many existing stats represent a value with units (e.g., microseconds), this was not always programatically determinable. In order to correctly expose stats for Prometheus, stats need to be scaled to base units (microseconds->seconds, kilobytes->bytes) and appropriately suffixed ( https://prometheus.io/docs/practices/naming/ ). This patch introduces Unit types which can be used to normalise values and determine the correct suffix. Units are not yet used in this patch, but will in future be used in metric definitions. Change-Id: I656f13406e8b039e54c84bdb464c45012ee92c1f Reviewed-on: http://review.couchbase.org/c/kv_engine/+/128633 Reviewed-by: Dave Rigby <[email protected]> Tested-by: Build Bot <[email protected]>
1 parent 1c2049d commit b3777d0

File tree

2 files changed

+211
-0
lines changed

2 files changed

+211
-0
lines changed

‎engines/ep/tests/module_tests/stats_test.cc‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
#include <memcached/server_cookie_iface.h>
4040
#include <programs/engine_testapp/mock_cookie.h>
4141
#include <programs/engine_testapp/mock_server.h>
42+
#include <statistics/stat_units.h>
4243

4344
#include <functional>
4445
#include <thread>
@@ -262,6 +263,78 @@ TEST_F(StatTest, HdrHistogramStatExpansion) {
262263
destroy_mock_cookie(cookie);
263264
}
264265

266+
TEST_F(StatTest, UnitNormalisation) {
267+
// check that metric units simplify the given value to the
268+
// expected "base" representation
269+
// e.g., if a stat has the value `456`, and is tracking nanoseconds,
270+
// units::nanoseconds should normalise that to
271+
// 0.000000456 - like so
272+
using namespace cb::stats;
273+
EXPECT_NEAR(0.000000456, units::nanoseconds.toBaseUnit(456), 0.000000001);
274+
// This will be used to ensure stats are exposed to Prometheus
275+
// in consistent base units, as recommended by their best practices
276+
// https://prometheus.io/docs/practices/naming/#base-units
277+
278+
// Generic units do not encode a scaling, they're
279+
// just a stand-in where no better unit applies
280+
EXPECT_EQ(1, units::none.toBaseUnit(1));
281+
EXPECT_EQ(1, units::count.toBaseUnit(1));
282+
283+
// Percent normalises [0,100] to [0.0,1.0]
284+
EXPECT_EQ(0.5, units::percent.toBaseUnit(50));
285+
EXPECT_EQ(0.5, units::ratio.toBaseUnit(0.5));
286+
287+
// time units normalise to seconds
288+
EXPECT_EQ(1.0 / 1000000000, units::nanoseconds.toBaseUnit(1));
289+
EXPECT_EQ(1.0 / 1000000, units::microseconds.toBaseUnit(1));
290+
EXPECT_EQ(1.0 / 1000, units::milliseconds.toBaseUnit(1));
291+
EXPECT_EQ(1, units::seconds.toBaseUnit(1));
292+
EXPECT_EQ(60, units::minutes.toBaseUnit(1));
293+
EXPECT_EQ(60 * 60, units::hours.toBaseUnit(1));
294+
EXPECT_EQ(60 * 60 * 24, units::days.toBaseUnit(1));
295+
296+
// bits normalise to bytes, as do all byte units
297+
EXPECT_EQ(0.5, units::bits.toBaseUnit(4));
298+
EXPECT_EQ(1, units::bits.toBaseUnit(8));
299+
EXPECT_EQ(1, units::bytes.toBaseUnit(1));
300+
EXPECT_EQ(1000, units::kilobytes.toBaseUnit(1));
301+
EXPECT_EQ(1000000, units::megabytes.toBaseUnit(1));
302+
EXPECT_EQ(1000000000, units::gigabytes.toBaseUnit(1));
303+
}
304+
305+
TEST_F(StatTest, UnitSuffix) {
306+
// check that metric units report the correct suffix for their
307+
// base unit, matching Prometheus recommendations in
308+
// https://prometheus.io/docs/practices/naming/#metric-names
309+
310+
using namespace cb::stats;
311+
312+
for (const auto& unit : {units::none, units::count}) {
313+
EXPECT_EQ("", unit.getSuffix());
314+
}
315+
316+
for (const auto& unit : {units::percent, units::ratio}) {
317+
EXPECT_EQ("_ratio", unit.getSuffix());
318+
}
319+
320+
for (const auto& unit : {units::microseconds,
321+
units::milliseconds,
322+
units::seconds,
323+
units::minutes,
324+
units::hours,
325+
units::days}) {
326+
EXPECT_EQ("_seconds", unit.getSuffix());
327+
}
328+
329+
for (const auto& unit : {units::bits,
330+
units::bytes,
331+
units::kilobytes,
332+
units::megabytes,
333+
units::gigabytes}) {
334+
EXPECT_EQ("_bytes", unit.getSuffix());
335+
}
336+
}
337+
265338
TEST_P(DatatypeStatTest, datatypesInitiallyZero) {
266339
// Check that the datatype stats initialise to 0
267340
auto vals = get_stat(nullptr);

‎include/statistics/stat_units.h‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2+
/*
3+
* Copyright 2020 Couchbase, Inc
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
#pragma once
19+
20+
#include <folly/lang/Assume.h>
21+
#include <gsl/gsl>
22+
23+
#include <array>
24+
#include <ratio>
25+
#include <string>
26+
27+
namespace cb::stats {
28+
29+
// Enum of the relevant base units used by convention in Prometheus
30+
enum class BaseUnit {
31+
None, // used for text stats where units aren't relevant
32+
Count, // Generic count which does not meet a better unit
33+
Seconds,
34+
Bytes,
35+
Ratio,
36+
};
37+
38+
/**
39+
* Type representing a unit (kilobytes, microseconds etc.).
40+
* This is encoded as a BaseUnit (see above) and a scaling factor stored as
41+
* two integers (see std::ratio).
42+
*
43+
* Thus,
44+
* kilobytes -> bytes, 1000/1
45+
* microseconds -> seconds, 1/1000000
46+
*
47+
* Given a numerical value in this unit, the scaling factor indicates
48+
* how to convert the value back to the base unit.
49+
*
50+
* 10 kilobytes
51+
* 10 * 1000 / 1
52+
* -> 10000 bytes
53+
*
54+
* This can be performed using the toBaseUnit method -
55+
*
56+
* Unit(std::kilo{}, BaseUnit::Bytes).toBaseUnit(10) == 10000
57+
*
58+
* Units will usually be used through the constexpr values
59+
* in namespace units
60+
*
61+
* units::kilobytes.toBaseUnit(10) == 10000
62+
*/
63+
class Unit {
64+
public:
65+
explicit constexpr Unit(BaseUnit baseUnit)
66+
: Unit(std::ratio<1>{}, baseUnit) {
67+
}
68+
69+
template <class RatioType>
70+
constexpr Unit(RatioType ratio, BaseUnit baseUnit)
71+
: numerator(gsl::narrow_cast<int64_t>(RatioType::num)),
72+
denominator(gsl::narrow_cast<int64_t>(RatioType::den)),
73+
baseUnit(baseUnit) {
74+
}
75+
76+
/**
77+
* Scale a value of the current unit (e.g., milliseconds) to the base
78+
* unit (e.g., seconds).
79+
*/
80+
[[nodiscard]] constexpr double toBaseUnit(double value) const {
81+
return (value * numerator) / denominator;
82+
}
83+
84+
[[nodiscard]] std::string_view getSuffix() const {
85+
using namespace std::string_view_literals;
86+
switch (baseUnit) {
87+
case BaseUnit::None:
88+
case BaseUnit::Count:
89+
return ""sv;
90+
case BaseUnit::Seconds:
91+
return "_seconds"sv;
92+
case BaseUnit::Bytes:
93+
return "_bytes"sv;
94+
case BaseUnit::Ratio:
95+
return "_ratio"sv;
96+
}
97+
folly::assume_unreachable();
98+
}
99+
100+
private:
101+
int64_t numerator;
102+
int64_t denominator;
103+
BaseUnit baseUnit;
104+
};
105+
106+
namespace units {
107+
constexpr Unit none{std::ratio<1>{}, BaseUnit::None};
108+
109+
constexpr Unit count{std::ratio<1>{}, BaseUnit::Count};
110+
111+
// floating point between 0 and 1, this is already in the correct base unit
112+
constexpr Unit ratio{std::ratio<1>{}, BaseUnit::Ratio};
113+
// percents should be scaled down to ratios for Prometheus
114+
constexpr Unit percent{std::ratio<1, 100>{}, BaseUnit::Ratio};
115+
116+
// time units
117+
constexpr Unit nanoseconds{std::nano{}, BaseUnit::Seconds};
118+
constexpr Unit microseconds{std::micro{}, BaseUnit::Seconds};
119+
constexpr Unit milliseconds{std::milli{}, BaseUnit::Seconds};
120+
constexpr Unit seconds{std::ratio<1>{}, BaseUnit::Seconds};
121+
constexpr Unit minutes{std::ratio<60>{}, BaseUnit::Seconds};
122+
constexpr Unit hours{std::ratio<60 * 60>{}, BaseUnit::Seconds};
123+
constexpr Unit days{std::ratio<60 * 60 * 24>{}, BaseUnit::Seconds};
124+
125+
// Note: std ratio definitions are powers of 10, not 2
126+
// e.g., kilo = std::ratio<1000, 1>
127+
// so 1 kilobyte = 1000 bytes
128+
// if powers of 2 are explicitly needed, IEC prefixes
129+
// (kibi, gibi, tebi) could easily be defined.
130+
131+
// byte units
132+
constexpr Unit bits{std::ratio<1, 8>{}, BaseUnit::Bytes};
133+
constexpr Unit bytes{std::ratio<1>{}, BaseUnit::Bytes};
134+
constexpr Unit kilobytes{std::kilo{}, BaseUnit::Bytes};
135+
constexpr Unit megabytes{std::mega{}, BaseUnit::Bytes};
136+
constexpr Unit gigabytes{std::giga{}, BaseUnit::Bytes};
137+
} // namespace units
138+
} // namespace cb::stats

0 commit comments

Comments
 (0)