Skip to content

Commit 12bae97

Browse files
committed
New addon -- KafkaMetrics
1 parent d7ccbee commit 12bae97

File tree

5 files changed

+773
-4
lines changed

5 files changed

+773
-4
lines changed

.github/workflows/kafka_api_ci_tests.yml

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ on:
77
- master
88

99
env:
10-
KAFKA_SRC_LINK: https://apache.newfountain.nl/kafka/2.7.1/kafka_2.13-2.7.1.tgz
10+
KAFKA_SRC_LINK: https://archive.apache.org/dist/kafka/2.8.1/kafka_2.13-2.8.1.tgz
1111
CPU_CORE_NUM: 2
1212
LIBRDKAFKA_VERSION: 1.7.0
1313
BUILD_SUB_DIR: build/sub-build
@@ -134,7 +134,11 @@ jobs:
134134
make -j${CPU_CORE_NUM} && sudo make install
135135
cd ../
136136
137-
# 5. Install tools to generate document
137+
# 5. Install rapidjson (for `addons/KafkaMetrics.h`)
138+
wget -nv https://github.com/Tencent/rapidjson/archive/refs/tags/v1.1.0.tar.gz
139+
tar -xzf v1.1.0.tar.gz
140+
141+
# 6. Install tools to generate document
138142
if [ ${GENERATE_DOC} ]; then
139143
sudo apt install -y python3-pip
140144
sudo pip3 install markdown
@@ -184,11 +188,13 @@ jobs:
184188
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_GEN_DOC=ON"
185189
fi
186190
191+
export RAPIDJSON_INCLUDE_DIRS=`pwd`/rapidjson-1.1.0/include
187192
env CXX=${BUILD_CXX} cmake ../.. ${CMAKE_CXX_STANDARD} ${CMAKE_BUILD_TYPE} ${BUILD_OPTION}
188193
189194
- name: Build
190195
run: |
191196
cd ${BUILD_SUB_DIR}
197+
192198
make -j${CPU_CORE_NUM} VERBOSE=1
193199
194200
- name: Install
@@ -275,6 +281,9 @@ jobs:
275281
cp -v "C:\VCPKG\INSTALLED\x86-windows\bin\boost_program_options-vc142-mt-x32-1_77.dll" "C:\VCPKG\INSTALLED\x86-windows\bin\boost_program_options.dll"
276282
cp -v "C:\VCPKG\INSTALLED\x86-windows\bin\boost_program_options-vc142-mt-x32-1_77.pdb" "C:\VCPKG\INSTALLED\x86-windows\bin\boost_program_options.pdb"
277283
284+
# Install rapidjson
285+
vcpkg install rapidjson
286+
278287
vcpkg integrate install
279288
280289
- name: Config
@@ -303,8 +312,8 @@ jobs:
303312
tree "tests"
304313
305314
# Install kafka
306-
Invoke-WebRequest -Uri $Env:KAFKA_SRC_LINK -OutFile kafka_2.13-2.7.1.tgz
307-
tar xvzf kafka_2.13-2.7.1.tgz
315+
Invoke-WebRequest -Uri $Env:KAFKA_SRC_LINK -OutFile kafka_2.13-2.8.1.tgz
316+
tar xvzf kafka_2.13-2.8.1.tgz
308317
309318
ctest -VV -L $Env:TEST_LABELS
310319

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ Eventually, we worked out the ***modern-cpp-kafka***, -- a header-only library t
7171

7272
* `SASL_LIBRARYDIR`/`SASL_LIBRARY` -- if SASL connection support is wanted
7373

74+
* `RAPIDJSON_INCLUDE_DIRS` -- `addons/KafkaMetrics` requires **rapidjson** headers
75+
7476
* Create an empty directory for the build, and `cd` to it
7577

7678
* Build commands
Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
#pragma once
2+
3+
#include "kafka/Project.h"
4+
5+
// https://github.com/Tencent/rapidjson/releases/tag/v1.1.0
6+
#include "rapidjson/document.h"
7+
#include "rapidjson/stringbuffer.h"
8+
#include "rapidjson/writer.h"
9+
10+
#include <algorithm>
11+
#include <iostream>
12+
#include <sstream>
13+
#include <stdexcept>
14+
#include <string>
15+
#include <utility>
16+
#include <vector>
17+
18+
19+
namespace KAFKA_API {
20+
21+
/**
22+
* \brief Helps to parse the metrics string with JSON format.
23+
*/
24+
class KafkaMetrics
25+
{
26+
public:
27+
/**
28+
* \brief Initilize with the metrics string.
29+
*/
30+
explicit KafkaMetrics(std::string jsonMetrics);
31+
32+
static const constexpr char* WILDCARD = "*";
33+
34+
using KeysType = std::vector<std::string>;
35+
36+
/**
37+
* \brief The matched keys (for wildcards) and the value.
38+
*/
39+
template<typename ValueType>
40+
using ResultsType = std::vector<std::pair<KeysType, ValueType>>;
41+
42+
/**
43+
* \brief Get integer value(s) for the specified metrics.
44+
* Note: the wildcard ("*") is supported.
45+
*/
46+
ResultsType<std::int64_t> getInt(const KeysType& keys) { return get<std::int64_t>(keys); }
47+
48+
/**
49+
* \brief Get string value(s) for the specified metrics.
50+
* Note: the wildcard ("*") is supported.
51+
*/
52+
ResultsType<std::string> getString(const KeysType& keys) { return get<std::string>(keys); }
53+
54+
static std::string toString(const KafkaMetrics::KeysType& keys);
55+
56+
template<typename ValueType>
57+
static std::string toString(const KafkaMetrics::ResultsType<ValueType>& results);
58+
59+
private:
60+
template<typename ValueType>
61+
ResultsType<ValueType> get(const KeysType& keys);
62+
63+
template<typename ValueType>
64+
void getResults(ResultsType<ValueType>& results,
65+
KeysType& keysForWildcards,
66+
rapidjson::Value::ConstMemberIterator iter,
67+
KeysType::const_iterator keysToParse,
68+
KeysType::const_iterator keysEnd);
69+
70+
template<typename ValueType>
71+
static ValueType getValue(rapidjson::Value::ConstMemberIterator iter);
72+
73+
#if COMPILER_SUPPORTS_CPP_17
74+
std::string _decodeBuf;
75+
#else
76+
std::vector<char> _decodeBuf;
77+
#endif
78+
rapidjson::Document _jsonDoc;
79+
};
80+
81+
inline
82+
KafkaMetrics::KafkaMetrics(std::string jsonMetrics)
83+
#if COMPILER_SUPPORTS_CPP_17
84+
: _decodeBuf(std::move(jsonMetrics))
85+
#else
86+
: _decodeBuf(jsonMetrics.cbegin(), jsonMetrics.cend() + 1)
87+
#endif
88+
{
89+
if (_jsonDoc.ParseInsitu(_decodeBuf.data()).HasParseError())
90+
{
91+
throw std::runtime_error("Failed to parse string with JSON format!");
92+
}
93+
}
94+
95+
template<>
96+
inline std::int64_t
97+
KafkaMetrics::getValue<std::int64_t>(rapidjson::Value::ConstMemberIterator iter)
98+
{
99+
return iter->value.GetInt();
100+
}
101+
102+
template<>
103+
inline std::string
104+
KafkaMetrics::getValue<std::string>(rapidjson::Value::ConstMemberIterator iter)
105+
{
106+
return iter->value.GetString();
107+
}
108+
109+
template<typename ValueType>
110+
inline KafkaMetrics::ResultsType<ValueType>
111+
KafkaMetrics::get(const KeysType& keys)
112+
{
113+
if (keys.empty()) throw std::invalid_argument("Input keys cannot be empty!");
114+
if (keys.front() == WILDCARD) throw std::invalid_argument("The first key cannot be wildcard!");
115+
if (keys.back() == WILDCARD) throw std::invalid_argument("The last key cannot be wildcard!");
116+
117+
ResultsType<ValueType> results;
118+
119+
rapidjson::Value::ConstMemberIterator iter = _jsonDoc.FindMember(keys.front().c_str());
120+
if (iter == _jsonDoc.MemberEnd()) return results;
121+
122+
if (keys.size() == 1)
123+
{
124+
if (std::is_same<ValueType, std::string>::value ? iter->value.IsString() : iter->value.IsInt())
125+
{
126+
results.emplace_back(KeysType{}, getValue<ValueType>(iter));
127+
}
128+
129+
return results;
130+
}
131+
132+
KeysType keysForWildcards;
133+
134+
getResults(results, keysForWildcards, iter, keys.cbegin() + 1, keys.cend());
135+
return results;
136+
}
137+
138+
template<typename ValueType>
139+
inline void
140+
KafkaMetrics::getResults(KafkaMetrics::ResultsType<ValueType>& results,
141+
KeysType& keysForWildcards,
142+
rapidjson::Value::ConstMemberIterator iter,
143+
KeysType::const_iterator keysToParse,
144+
KeysType::const_iterator keysEnd)
145+
{
146+
if (!iter->value.IsObject()) return;
147+
148+
const auto& key = *(keysToParse++);
149+
const bool isTheEnd = (keysToParse == keysEnd);
150+
151+
if (key == WILDCARD)
152+
{
153+
for (rapidjson::Value::ConstMemberIterator subIter = iter->value.MemberBegin(); subIter != iter->value.MemberEnd(); ++subIter)
154+
{
155+
KeysType newKeysForWildcards = keysForWildcards;
156+
newKeysForWildcards.emplace_back(subIter->name.GetString());
157+
158+
getResults(results, newKeysForWildcards, subIter, keysToParse, keysEnd);
159+
}
160+
}
161+
else
162+
{
163+
rapidjson::Value::ConstMemberIterator subIter = iter->value.FindMember(key.c_str());
164+
if (subIter == iter->value.MemberEnd()) return;
165+
166+
if (!isTheEnd)
167+
{
168+
getResults(results, keysForWildcards, subIter, keysToParse, keysEnd);
169+
}
170+
else if (std::is_same<ValueType, std::string>::value ? subIter->value.IsString() : subIter->value.IsInt())
171+
{
172+
results.emplace_back(keysForWildcards, getValue<ValueType>(subIter));
173+
}
174+
}
175+
}
176+
177+
inline std::string
178+
KafkaMetrics::toString(const KafkaMetrics::KeysType& keys)
179+
{
180+
std::string ret;
181+
182+
std::for_each(keys.cbegin(), keys.cend(),
183+
[&ret](const auto& key){ ret.append((ret.empty() ? std::string() : std::string(", ")) + "\"" + key + "\""); });
184+
185+
return ret;
186+
}
187+
188+
template<typename ValueType>
189+
inline std::string
190+
KafkaMetrics::toString(const KafkaMetrics::ResultsType<ValueType>& results)
191+
{
192+
std::ostringstream oss;
193+
bool isTheFirstOne = true;
194+
195+
std::for_each(results.cbegin(), results.cend(),
196+
[&oss, &isTheFirstOne](const auto& result) {
197+
const auto keysString = toString(result.first);
198+
199+
oss << (isTheFirstOne ? (isTheFirstOne = false, "") : ", ")
200+
<< (keysString.empty() ? "" : (std::string("[") + keysString + "]:"));
201+
oss << (std::is_same<ValueType, std::string>::value ? "\"" : "") << result.second << (std::is_same<ValueType, std::string>::value ? "\"" : "");
202+
});
203+
204+
return oss.str();
205+
}
206+
207+
} // end of KAFKA_API
208+

tests/unit/CMakeLists.txt

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,25 @@
11
project("kafka-unit-test")
22

3+
#---------------------------
4+
# rapidjson
5+
#---------------------------
6+
if (DEFINED ENV{RAPIDJSON_INCLUDE_DIRS})
7+
set(RAPIDJSON_INCLUDE_DIRS $ENV{RAPIDJSON_INCLUDE_DIRS})
8+
else ()
9+
find_package(rapidjson REQUIRED)
10+
if (NOT RAPIDJSON_INCLUDE_DIRS)
11+
message(FATAL_ERROR "Rapidjson not found!")
12+
endif ()
13+
endif ()
14+
15+
message(STATUS "rapidjson include directory: ${RAPIDJSON_INCLUDE_DIRS}")
16+
17+
318
# Target
419
file(GLOB TEST_SRCS *.cc)
520

21+
include_directories(${PROJECT_NAME} SYSTEM INTERFACE ${RAPIDJSON_INCLUDE_DIRS})
22+
623
add_executable("${PROJECT_NAME}" ${TEST_SRCS})
724
target_link_libraries("${PROJECT_NAME}" modern-cpp-kafka-api gtest gtest_main)
825

0 commit comments

Comments
 (0)